use LlamaTokenizer

spiece.model → tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1e2fccdac2794faadad487a5bf76018f693c71361ba7eb088bcaf9fe9ee90c23
-size 1831863
+oid sha256:008293028e1a9d9a1038d9b63d989a2319797dfeaa03f171093a57b33a3a8277
+size 1831879

tokenizer_config.json

@@ -12,8 +12,32 @@
   "padding_side": "left",
   "sp_model_kwargs": {},
   "special_tokens_map_file": null,
-  "tokenizer_class": "T5Tokenizer",
+  "tokenizer_class": "LlamaTokenizer",
   "added_tokens_decoder": {
+    "4": {
+      "content": "<sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "<cls>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
     "7": {
       "content": "<|system|>",
       "lstrip": false,
@@ -102,5 +126,10 @@
       "single_word": false,
       "special": false
     }
-  }
+  },
+  "add_prefix_space": false,
+  "add_dummy_prefix_space": false,
+  "legacy": false,
+  "add_bos_token": false,
+  "add_eos_token": true
 }