sr1ncvs commited on
Commit
f52b36a
1 Parent(s): 15440de

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +3 -3
  2. tokenizer_config.json +4 -4
  3. vocab.json +9 -10
added_tokens.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "</s>": 102,
3
- "<s>": 101,
4
- "[PAD]": 100
5
  }
 
1
  {
2
+ "</s>": 101,
3
+ "<s>": 100,
4
+ "[PAD]": 99
5
  }
tokenizer_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "added_tokens_decoder": {
3
- "99": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
@@ -8,7 +8,7 @@
8
  "single_word": false,
9
  "special": false
10
  },
11
- "100": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
@@ -16,7 +16,7 @@
16
  "single_word": false,
17
  "special": false
18
  },
19
- "101": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": false,
@@ -24,7 +24,7 @@
24
  "single_word": false,
25
  "special": true
26
  },
27
- "102": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": false,
 
1
  {
2
  "added_tokens_decoder": {
3
+ "98": {
4
  "content": "[UNK]",
5
  "lstrip": true,
6
  "normalized": false,
 
8
  "single_word": false,
9
  "special": false
10
  },
11
+ "99": {
12
  "content": "[PAD]",
13
  "lstrip": true,
14
  "normalized": false,
 
16
  "single_word": false,
17
  "special": false
18
  },
19
+ "100": {
20
  "content": "<s>",
21
  "lstrip": false,
22
  "normalized": false,
 
24
  "single_word": false,
25
  "special": true
26
  },
27
+ "101": {
28
  "content": "</s>",
29
  "lstrip": false,
30
  "normalized": false,
vocab.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "&": 1,
3
- "[PAD]": 100,
4
- "[UNK]": 99,
5
  "a": 2,
6
  "b": 3,
7
  "c": 4,
@@ -92,12 +92,11 @@
92
  "्": 89,
93
  "क़": 90,
94
  "ख़": 91,
95
- "": 92,
96
- "": 93,
97
- "": 94,
98
- "": 95,
99
- "": 96,
100
- "": 97,
101
- "": 98,
102
- "’": 99
103
  }
 
1
  {
2
  "&": 1,
3
+ "[PAD]": 99,
4
+ "[UNK]": 98,
5
  "a": 2,
6
  "b": 3,
7
  "c": 4,
 
92
  "्": 89,
93
  "क़": 90,
94
  "ख़": 91,
95
+ "": 92,
96
+ "": 93,
97
+ "": 94,
98
+ "": 95,
99
+ "": 96,
100
+ "": 97,
101
+ "": 98
 
102
  }