Upload tokenizer

added_tokens.json

@@ -0,0 +1,98 @@
+{
+  "<extra_id_0>": 36095,
+  "<extra_id_10>": 36085,
+  "<extra_id_11>": 36084,
+  "<extra_id_12>": 36083,
+  "<extra_id_13>": 36082,
+  "<extra_id_14>": 36081,
+  "<extra_id_15>": 36080,
+  "<extra_id_16>": 36079,
+  "<extra_id_17>": 36078,
+  "<extra_id_18>": 36077,
+  "<extra_id_19>": 36076,
+  "<extra_id_1>": 36094,
+  "<extra_id_20>": 36075,
+  "<extra_id_21>": 36074,
+  "<extra_id_22>": 36073,
+  "<extra_id_23>": 36072,
+  "<extra_id_24>": 36071,
+  "<extra_id_25>": 36070,
+  "<extra_id_26>": 36069,
+  "<extra_id_27>": 36068,
+  "<extra_id_28>": 36067,
+  "<extra_id_29>": 36066,
+  "<extra_id_2>": 36093,
+  "<extra_id_30>": 36065,
+  "<extra_id_31>": 36064,
+  "<extra_id_32>": 36063,
+  "<extra_id_33>": 36062,
+  "<extra_id_34>": 36061,
+  "<extra_id_35>": 36060,
+  "<extra_id_36>": 36059,
+  "<extra_id_37>": 36058,
+  "<extra_id_38>": 36057,
+  "<extra_id_39>": 36056,
+  "<extra_id_3>": 36092,
+  "<extra_id_40>": 36055,
+  "<extra_id_41>": 36054,
+  "<extra_id_42>": 36053,
+  "<extra_id_43>": 36052,
+  "<extra_id_44>": 36051,
+  "<extra_id_45>": 36050,
+  "<extra_id_46>": 36049,
+  "<extra_id_47>": 36048,
+  "<extra_id_48>": 36047,
+  "<extra_id_49>": 36046,
+  "<extra_id_4>": 36091,
+  "<extra_id_50>": 36045,
+  "<extra_id_51>": 36044,
+  "<extra_id_52>": 36043,
+  "<extra_id_53>": 36042,
+  "<extra_id_54>": 36041,
+  "<extra_id_55>": 36040,
+  "<extra_id_56>": 36039,
+  "<extra_id_57>": 36038,
+  "<extra_id_58>": 36037,
+  "<extra_id_59>": 36036,
+  "<extra_id_5>": 36090,
+  "<extra_id_60>": 36035,
+  "<extra_id_61>": 36034,
+  "<extra_id_62>": 36033,
+  "<extra_id_63>": 36032,
+  "<extra_id_64>": 36031,
+  "<extra_id_65>": 36030,
+  "<extra_id_66>": 36029,
+  "<extra_id_67>": 36028,
+  "<extra_id_68>": 36027,
+  "<extra_id_69>": 36026,
+  "<extra_id_6>": 36089,
+  "<extra_id_70>": 36025,
+  "<extra_id_71>": 36024,
+  "<extra_id_72>": 36023,
+  "<extra_id_73>": 36022,
+  "<extra_id_74>": 36021,
+  "<extra_id_75>": 36020,
+  "<extra_id_76>": 36019,
+  "<extra_id_77>": 36018,
+  "<extra_id_78>": 36017,
+  "<extra_id_79>": 36016,
+  "<extra_id_7>": 36088,
+  "<extra_id_80>": 36015,
+  "<extra_id_81>": 36014,
+  "<extra_id_82>": 36013,
+  "<extra_id_83>": 36012,
+  "<extra_id_84>": 36011,
+  "<extra_id_85>": 36010,
+  "<extra_id_86>": 36009,
+  "<extra_id_87>": 36008,
+  "<extra_id_88>": 36007,
+  "<extra_id_89>": 36006,
+  "<extra_id_8>": 36087,
+  "<extra_id_90>": 36005,
+  "<extra_id_91>": 36004,
+  "<extra_id_92>": 36003,
+  "<extra_id_93>": 36002,
+  "<extra_id_94>": 36001,
+  "<extra_id_95>": 36000,
+  "<extra_id_9>": 36086
+}

special_tokens_map.json

@@ -97,7 +97,25 @@
     "<extra_id_94>",
     "<extra_id_95>"
   ],
-  "eos_token": "</s>",
-  "pad_token": "<pad>",
-  "unk_token": "<unk>"
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<pad>",
@@ -894,6 +895,7 @@
   "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 96,
+  "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},