raygx commited on
Commit
4e920ad
1 Parent(s): ffba65e

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -2
tokenizer.json CHANGED
@@ -18,8 +18,8 @@
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
21
- "normalized": false,
22
- "special": true
23
  },
24
  {
25
  "id": 50001,
 
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
21
+ "normalized": true,
22
+ "special": false
23
  },
24
  {
25
  "id": 50001,