abhinavkulkarni commited on
Commit
8ca5acf
β€’
1 Parent(s): 5970d6e

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +2 -4
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,6 +1,4 @@
1
  {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
  "additional_special_tokens": [
5
  "▁<PRE>",
6
  "▁<MID>",
@@ -32,7 +30,6 @@
32
  "pad_token": null,
33
  "prefix_token": "▁<PRE>",
34
  "sp_model_kwargs": {},
35
- "suffix_first": false,
36
  "suffix_token": "▁<SUF>",
37
  "tokenizer_class": "CodeLlamaTokenizer",
38
  "unk_token": {
@@ -42,5 +39,6 @@
42
  "normalized": true,
43
  "rstrip": false,
44
  "single_word": false
45
- }
 
46
  }
 
1
  {
 
 
2
  "additional_special_tokens": [
3
  "▁<PRE>",
4
  "▁<MID>",
 
30
  "pad_token": null,
31
  "prefix_token": "▁<PRE>",
32
  "sp_model_kwargs": {},
 
33
  "suffix_token": "▁<SUF>",
34
  "tokenizer_class": "CodeLlamaTokenizer",
35
  "unk_token": {
 
39
  "normalized": true,
40
  "rstrip": false,
41
  "single_word": false
42
+ },
43
+ "use_default_system_prompt": false
44
  }