Upload tokenizer.json
Browse files- tokenizer.json +18 -6
tokenizer.json
CHANGED
|
@@ -1,7 +1,19 @@
|
|
| 1 |
{
|
| 2 |
"version": "1.0",
|
| 3 |
-
"truncation":
|
| 4 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
"added_tokens": [
|
| 6 |
{
|
| 7 |
"id": 0,
|
|
@@ -944,8 +956,8 @@
|
|
| 944 |
{
|
| 945 |
"type": "Metaspace",
|
| 946 |
"replacement": "▁",
|
| 947 |
-
"
|
| 948 |
-
"
|
| 949 |
}
|
| 950 |
]
|
| 951 |
},
|
|
@@ -1006,8 +1018,8 @@
|
|
| 1006 |
"decoder": {
|
| 1007 |
"type": "Metaspace",
|
| 1008 |
"replacement": "▁",
|
| 1009 |
-
"
|
| 1010 |
-
"
|
| 1011 |
},
|
| 1012 |
"model": {
|
| 1013 |
"type": "Unigram",
|
|
|
|
| 1 |
{
|
| 2 |
"version": "1.0",
|
| 3 |
+
"truncation": {
|
| 4 |
+
"direction": "Right",
|
| 5 |
+
"max_length": 50,
|
| 6 |
+
"strategy": "LongestFirst",
|
| 7 |
+
"stride": 0
|
| 8 |
+
},
|
| 9 |
+
"padding": {
|
| 10 |
+
"strategy": "BatchLongest",
|
| 11 |
+
"direction": "Right",
|
| 12 |
+
"pad_to_multiple_of": null,
|
| 13 |
+
"pad_id": 0,
|
| 14 |
+
"pad_type_id": 0,
|
| 15 |
+
"pad_token": "<pad>"
|
| 16 |
+
},
|
| 17 |
"added_tokens": [
|
| 18 |
{
|
| 19 |
"id": 0,
|
|
|
|
| 956 |
{
|
| 957 |
"type": "Metaspace",
|
| 958 |
"replacement": "▁",
|
| 959 |
+
"add_prefix_space": true,
|
| 960 |
+
"prepend_scheme": "always"
|
| 961 |
}
|
| 962 |
]
|
| 963 |
},
|
|
|
|
| 1018 |
"decoder": {
|
| 1019 |
"type": "Metaspace",
|
| 1020 |
"replacement": "▁",
|
| 1021 |
+
"add_prefix_space": true,
|
| 1022 |
+
"prepend_scheme": "always"
|
| 1023 |
},
|
| 1024 |
"model": {
|
| 1025 |
"type": "Unigram",
|