danielhanchen commited on
Commit
b3c6c51
1 Parent(s): 2138b9d

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +7 -1
  2. tokenizer.json +2 -2
special_tokens_map.json CHANGED
@@ -21,5 +21,11 @@
21
  "rstrip": false,
22
  "single_word": false
23
  },
24
- "pad_token": "<|PAD_TOKEN|>"
 
 
 
 
 
 
25
  }
 
21
  "rstrip": false,
22
  "single_word": false
23
  },
24
+ "pad_token": {
25
+ "content": "<|PAD_TOKEN|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
  }
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:29503e9663ee5b7f65e04bb44f7225f2607fcd7995d7803632f42156db47ecfe
3
- size 7031863
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fab42efe8d17406525a9154b728cf9e957629a8ed7ce997770efdd71128c6a1a
3
+ size 11422086