ashishsr commited on
Commit
ede9f0b
1 Parent(s): 1bcf03b

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +8 -0
tokenizer_config.json CHANGED
@@ -25,6 +25,14 @@
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
 
 
 
 
 
 
 
 
28
  }
29
  },
30
  "bos_token": {
 
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
+ },
29
+ "3": {
30
+ "content": "|<pad>|",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
  }
37
  },
38
  "bos_token": {