Upload tokenizer

added_tokens.json

@@ -1,36 +1,24 @@
 {
-  "ADJ": 49152,
-  "ADP": 49153,
-  "ADV": 49154,
-  "AUX": 49155,
-  "INTJ": 49156,
-  "JJ": 49165,
-  "JJR": 49166,
-  "JJS": 49167,
-  "NNP": 49169,
-  "NNPS": 49170,
-  "NNS": 49168,
-  "NOUN": 49157,
-  "PART": 49158,
-  "PDT": 49171,
-  "PRON": 49159,
-  "PROPN": 49160,
-  "PRP": 49172,
-  "PRP$": 49173,
-  "PUNCT": 49161,
-  "RBR": 49174,
-  "RBS": 49175,
-  "SCONJ": 49162,
-  "SYM": 49163,
-  "UH": 49176,
-  "VB": 49177,
-  "VBD": 49178,
-  "VBG": 49179,
-  "VBN": 49180,
-  "VBP": 49181,
-  "VBZ": 49182,
-  "VERB": 49164,
-  "WDT": 49183,
-  "WP$": 49184,
-  "WRB": 49185
+  "JJ": 49157,
+  "JJR": 49170,
+  "JJS": 49169,
+  "NNP": 49162,
+  "NNPS": 49168,
+  "NNS": 49161,
+  "PDT": 49164,
+  "PRP": 49166,
+  "PRP$": 49152,
+  "RBR": 49159,
+  "RBS": 49165,
+  "SYM": 49171,
+  "UH": 49172,
+  "VB": 49154,
+  "VBD": 49160,
+  "VBG": 49153,
+  "VBN": 49156,
+  "VBP": 49155,
+  "VBZ": 49158,
+  "WDT": 49173,
+  "WP$": 49167,
+  "WRB": 49163
 }

special_tokens_map.json

@@ -1,41 +1,50 @@
 {
   "additional_special_tokens": [
-    "CC",
-    "CD",
-    "DT",
-    "EX",
+    "PRP$",
+    "VBG",
     "FW",
-    "IN",
+    "VB",
+    "POS",
+    "''",
+    "VBP",
+    "VBN",
     "JJ",
-    "JJR",
-    "JJS",
-    "LS",
-    "MD",
+    "WP",
+    "VBZ",
+    "DT",
+    "RP",
+    "$",
     "NN",
+    ")",
+    "(",
+    "RBR",
+    "VBD",
+    ",",
+    ".",
+    "TO",
+    "LS",
+    "RB",
+    ":",
     "NNS",
     "NNP",
-    "NNPS",
+    "``",
+    "WRB",
+    "CC",
     "PDT",
-    "POS",
-    "PRP",
-    "PRP$",
-    "RB",
-    "RBR",
     "RBS",
-    "RP",
+    "PRP",
+    "CD",
+    "EX",
+    "IN",
+    "WP$",
+    "MD",
+    "NNPS",
+    "--",
+    "JJS",
+    "JJR",
     "SYM",
-    "TO",
     "UH",
-    "VB",
-    "VBD",
-    "VBG",
-    "VBN",
-    "VBP",
-    "VBZ",
-    "WDT",
-    "WP",
-    "WP$",
-    "WRB"
+    "WDT"
   ],
   "bos_token": {
     "content": "<|endoftext|>",
@@ -51,7 +60,13 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|endoftext|>",
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<|endoftext|>",
     "lstrip": false,

tokenizer.json

@@ -157,8 +157,8 @@
       "special": true
     },
     {
-      "id": 2113,
-      "content": "IN",
+      "id": 20,
+      "content": "$",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -166,8 +166,8 @@
       "special": true
     },
     {
-      "id": 5137,
-      "content": "CC",
+      "id": 24,
+      "content": "(",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -175,8 +175,8 @@
       "special": true
     },
     {
-      "id": 6998,
-      "content": "CD",
+      "id": 25,
+      "content": ")",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -184,8 +184,8 @@
       "special": true
     },
     {
-      "id": 9324,
-      "content": "EX",
+      "id": 28,
+      "content": ",",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -193,8 +193,8 @@
       "special": true
     },
     {
-      "id": 9815,
-      "content": "MD",
+      "id": 30,
+      "content": ".",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -202,8 +202,8 @@
       "special": true
     },
     {
-      "id": 10179,
-      "content": "TO",
+      "id": 42,
+      "content": ":",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -211,8 +211,8 @@
       "special": true
     },
     {
-      "id": 12041,
-      "content": "LS",
+      "id": 423,
+      "content": "--",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -220,8 +220,8 @@
       "special": true
     },
     {
-      "id": 17679,
-      "content": "DT",
+      "id": 1969,
+      "content": "``",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -229,8 +229,8 @@
       "special": true
     },
     {
-      "id": 19668,
-      "content": "NN",
+      "id": 2113,
+      "content": "IN",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -238,8 +238,8 @@
       "special": true
     },
     {
-      "id": 22259,
-      "content": "POS",
+      "id": 3816,
+      "content": "''",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -247,8 +247,8 @@
       "special": true
     },
     {
-      "id": 25398,
-      "content": "NUM",
+      "id": 5137,
+      "content": "CC",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -256,8 +256,8 @@
       "special": true
     },
     {
-      "id": 29482,
-      "content": "RP",
+      "id": 6998,
+      "content": "CD",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -265,8 +265,8 @@
       "special": true
     },
     {
-      "id": 35991,
-      "content": "FW",
+      "id": 9324,
+      "content": "EX",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -274,8 +274,8 @@
       "special": true
     },
     {
-      "id": 45780,
-      "content": "RB",
+      "id": 9815,
+      "content": "MD",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -283,8 +283,8 @@
       "special": true
     },
     {
-      "id": 47306,
-      "content": "WP",
+      "id": 10179,
+      "content": "TO",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -292,8 +292,8 @@
       "special": true
     },
     {
-      "id": 49152,
-      "content": "ADJ",
+      "id": 12041,
+      "content": "LS",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -301,8 +301,8 @@
       "special": true
     },
     {
-      "id": 49153,
-      "content": "ADP",
+      "id": 17679,
+      "content": "DT",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -310,8 +310,8 @@
       "special": true
     },
     {
-      "id": 49154,
-      "content": "ADV",
+      "id": 19668,
+      "content": "NN",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -319,8 +319,8 @@
       "special": true
     },
     {
-      "id": 49155,
-      "content": "AUX",
+      "id": 22259,
+      "content": "POS",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -328,8 +328,8 @@
       "special": true
     },
     {
-      "id": 49156,
-      "content": "INTJ",
+      "id": 29482,
+      "content": "RP",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -337,8 +337,8 @@
       "special": true
     },
     {
-      "id": 49157,
-      "content": "NOUN",
+      "id": 35991,
+      "content": "FW",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -346,8 +346,8 @@
       "special": true
     },
     {
-      "id": 49158,
-      "content": "PART",
+      "id": 45780,
+      "content": "RB",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -355,8 +355,8 @@
       "special": true
     },
     {
-      "id": 49159,
-      "content": "PRON",
+      "id": 47306,
+      "content": "WP",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -364,8 +364,8 @@
       "special": true
     },
     {
-      "id": 49160,
-      "content": "PROPN",
+      "id": 49152,
+      "content": "PRP$",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -373,8 +373,8 @@
       "special": true
     },
     {
-      "id": 49161,
-      "content": "PUNCT",
+      "id": 49153,
+      "content": "VBG",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -382,8 +382,8 @@
       "special": true
     },
     {
-      "id": 49162,
-      "content": "SCONJ",
+      "id": 49154,
+      "content": "VB",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -391,8 +391,8 @@
       "special": true
     },
     {
-      "id": 49163,
-      "content": "SYM",
+      "id": 49155,
+      "content": "VBP",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -400,8 +400,8 @@
       "special": true
     },
     {
-      "id": 49164,
-      "content": "VERB",
+      "id": 49156,
+      "content": "VBN",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -409,7 +409,7 @@
       "special": true
     },
     {
-      "id": 49165,
+      "id": 49157,
       "content": "JJ",
       "single_word": false,
       "lstrip": false,
@@ -418,26 +418,8 @@
       "special": true
     },
     {
-      "id": 49166,
-      "content": "JJR",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 49167,
-      "content": "JJS",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 49168,
-      "content": "NNS",
+      "id": 49158,
+      "content": "VBZ",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -445,8 +427,8 @@
       "special": true
     },
     {
-      "id": 49169,
-      "content": "NNP",
+      "id": 49159,
+      "content": "RBR",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -454,8 +436,8 @@
       "special": true
     },
     {
-      "id": 49170,
-      "content": "NNPS",
+      "id": 49160,
+      "content": "VBD",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -463,8 +445,8 @@
       "special": true
     },
     {
-      "id": 49171,
-      "content": "PDT",
+      "id": 49161,
+      "content": "NNS",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -472,8 +454,8 @@
       "special": true
     },
     {
-      "id": 49172,
-      "content": "PRP",
+      "id": 49162,
+      "content": "NNP",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -481,8 +463,8 @@
       "special": true
     },
     {
-      "id": 49173,
-      "content": "PRP$",
+      "id": 49163,
+      "content": "WRB",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -490,8 +472,8 @@
       "special": true
     },
     {
-      "id": 49174,
-      "content": "RBR",
+      "id": 49164,
+      "content": "PDT",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -499,7 +481,7 @@
       "special": true
     },
     {
-      "id": 49175,
+      "id": 49165,
       "content": "RBS",
       "single_word": false,
       "lstrip": false,
@@ -508,8 +490,8 @@
       "special": true
     },
     {
-      "id": 49176,
-      "content": "UH",
+      "id": 49166,
+      "content": "PRP",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -517,8 +499,8 @@
       "special": true
     },
     {
-      "id": 49177,
-      "content": "VB",
+      "id": 49167,
+      "content": "WP$",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -526,8 +508,8 @@
       "special": true
     },
     {
-      "id": 49178,
-      "content": "VBD",
+      "id": 49168,
+      "content": "NNPS",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -535,8 +517,8 @@
       "special": true
     },
     {
-      "id": 49179,
-      "content": "VBG",
+      "id": 49169,
+      "content": "JJS",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -544,8 +526,8 @@
       "special": true
     },
     {
-      "id": 49180,
-      "content": "VBN",
+      "id": 49170,
+      "content": "JJR",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -553,8 +535,8 @@
       "special": true
     },
     {
-      "id": 49181,
-      "content": "VBP",
+      "id": 49171,
+      "content": "SYM",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -562,8 +544,8 @@
       "special": true
     },
     {
-      "id": 49182,
-      "content": "VBZ",
+      "id": 49172,
+      "content": "UH",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -571,31 +553,13 @@
       "special": true
     },
     {
-      "id": 49183,
+      "id": 49173,
       "content": "WDT",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 49184,
-      "content": "WP$",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 49185,
-      "content": "WRB",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,

tokenizer_config.json

@@ -137,231 +137,231 @@
       "single_word": false,
       "special": true
     },
-    "2113": {
-      "content": "IN",
+    "20": {
+      "content": "$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "5137": {
-      "content": "CC",
+    "24": {
+      "content": "(",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "6998": {
-      "content": "CD",
+    "25": {
+      "content": ")",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "9324": {
-      "content": "EX",
+    "28": {
+      "content": ",",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "9815": {
-      "content": "MD",
+    "30": {
+      "content": ".",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "10179": {
-      "content": "TO",
+    "42": {
+      "content": ":",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "12041": {
-      "content": "LS",
+    "423": {
+      "content": "--",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "17679": {
-      "content": "DT",
+    "1969": {
+      "content": "``",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "19668": {
-      "content": "NN",
+    "2113": {
+      "content": "IN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "22259": {
-      "content": "POS",
+    "3816": {
+      "content": "''",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "25398": {
-      "content": "NUM",
+    "5137": {
+      "content": "CC",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "29482": {
-      "content": "RP",
+    "6998": {
+      "content": "CD",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "35991": {
-      "content": "FW",
+    "9324": {
+      "content": "EX",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "45780": {
-      "content": "RB",
+    "9815": {
+      "content": "MD",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "47306": {
-      "content": "WP",
+    "10179": {
+      "content": "TO",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49152": {
-      "content": "ADJ",
+    "12041": {
+      "content": "LS",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49153": {
-      "content": "ADP",
+    "17679": {
+      "content": "DT",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49154": {
-      "content": "ADV",
+    "19668": {
+      "content": "NN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49155": {
-      "content": "AUX",
+    "22259": {
+      "content": "POS",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49156": {
-      "content": "INTJ",
+    "29482": {
+      "content": "RP",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49157": {
-      "content": "NOUN",
+    "35991": {
+      "content": "FW",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49158": {
-      "content": "PART",
+    "45780": {
+      "content": "RB",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49159": {
-      "content": "PRON",
+    "47306": {
+      "content": "WP",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49160": {
-      "content": "PROPN",
+    "49152": {
+      "content": "PRP$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49161": {
-      "content": "PUNCT",
+    "49153": {
+      "content": "VBG",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49162": {
-      "content": "SCONJ",
+    "49154": {
+      "content": "VB",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49163": {
-      "content": "SYM",
+    "49155": {
+      "content": "VBP",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49164": {
-      "content": "VERB",
+    "49156": {
+      "content": "VBN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49165": {
+    "49157": {
       "content": "JJ",
       "lstrip": false,
       "normalized": false,
@@ -369,79 +369,63 @@
       "single_word": false,
       "special": true
     },
-    "49166": {
-      "content": "JJR",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "49167": {
-      "content": "JJS",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "49168": {
-      "content": "NNS",
+    "49158": {
+      "content": "VBZ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49169": {
-      "content": "NNP",
+    "49159": {
+      "content": "RBR",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49170": {
-      "content": "NNPS",
+    "49160": {
+      "content": "VBD",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49171": {
-      "content": "PDT",
+    "49161": {
+      "content": "NNS",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49172": {
-      "content": "PRP",
+    "49162": {
+      "content": "NNP",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49173": {
-      "content": "PRP$",
+    "49163": {
+      "content": "WRB",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49174": {
-      "content": "RBR",
+    "49164": {
+      "content": "PDT",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49175": {
+    "49165": {
       "content": "RBS",
       "lstrip": false,
       "normalized": false,
@@ -449,124 +433,117 @@
       "single_word": false,
       "special": true
     },
-    "49176": {
-      "content": "UH",
+    "49166": {
+      "content": "PRP",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49177": {
-      "content": "VB",
+    "49167": {
+      "content": "WP$",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49178": {
-      "content": "VBD",
+    "49168": {
+      "content": "NNPS",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49179": {
-      "content": "VBG",
+    "49169": {
+      "content": "JJS",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49180": {
-      "content": "VBN",
+    "49170": {
+      "content": "JJR",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49181": {
-      "content": "VBP",
+    "49171": {
+      "content": "SYM",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49182": {
-      "content": "VBZ",
+    "49172": {
+      "content": "UH",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49183": {
+    "49173": {
       "content": "WDT",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "49184": {
-      "content": "WP$",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "49185": {
-      "content": "WRB",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "additional_special_tokens": [
-    "CC",
-    "CD",
-    "DT",
-    "EX",
+    "PRP$",
+    "VBG",
     "FW",
-    "IN",
+    "VB",
+    "POS",
+    "''",
+    "VBP",
+    "VBN",
     "JJ",
-    "JJR",
-    "JJS",
-    "LS",
-    "MD",
+    "WP",
+    "VBZ",
+    "DT",
+    "RP",
+    "$",
     "NN",
+    ")",
+    "(",
+    "RBR",
+    "VBD",
+    ",",
+    ".",
+    "TO",
+    "LS",
+    "RB",
+    ":",
     "NNS",
     "NNP",
-    "NNPS",
+    "``",
+    "WRB",
+    "CC",
     "PDT",
-    "POS",
-    "PRP",
-    "PRP$",
-    "RB",
-    "RBR",
     "RBS",
-    "RP",
+    "PRP",
+    "CD",
+    "EX",
+    "IN",
+    "WP$",
+    "MD",
+    "NNPS",
+    "--",
+    "JJS",
+    "JJR",
     "SYM",
-    "TO",
     "UH",
-    "VB",
-    "VBD",
-    "VBG",
-    "VBN",
-    "VBP",
-    "VBZ",
-    "WDT",
-    "WP",
-    "WP$",
-    "WRB"
+    "WDT"
   ],
   "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": false,