devngho commited on
Commit
df4fe49
·
verified ·
1 Parent(s): ead403f

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -58,7 +58,7 @@
58
  "<|im_sep|>"
59
  ],
60
  "bos_token": "<s>",
61
- "chat_template": "{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'user') %}{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'assistant') %}{{'<|im_start|>assistant<|im_sep|>' + message['content'] + '<|im_end|>'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant<|im_sep|>' }}{% endif %}",
62
  "clean_up_tokenization_spaces": true,
63
  "eos_token": "</s>",
64
  "extra_special_tokens": {},
 
58
  "<|im_sep|>"
59
  ],
60
  "bos_token": "<s>",
61
+ "chat_template": "<s>{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'user') %}{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'assistant') %}{{'<|im_start|>assistant<|im_sep|>' + message['content'] + '<|im_end|>'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant<|im_sep|>' }}{% endif %}",
62
  "clean_up_tokenization_spaces": true,
63
  "eos_token": "</s>",
64
  "extra_special_tokens": {},