Update tokenizer_config.json
Browse files- tokenizer_config.json +0 -1
tokenizer_config.json
CHANGED
@@ -37,7 +37,6 @@
|
|
37 |
}
|
38 |
},
|
39 |
"bos_token": "<s>",
|
40 |
-
"chat_template": "{% if messages[0]['role'] == 'system' %}\n {% set loop_messages = messages[1:] %}\n {% set system_message = messages[0]['content'] %}\n{% else %}\n {% set loop_messages = messages %}\n {% set system_message = false %}\n{% endif %}\n{% if system_message != false and (loop_messages|length == 0 or loop_messages|length == 1 and loop_messages[0]['role'] != 'user') %}\n {{ raise_exception('The system prompt must be passed along with a user prompt.') }}\n{% endif %}\n{% for message in loop_messages %}\n {% if loop_messages|length > 1 and (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n {% endif %}\n {% if loop.index0 == 0 and system_message != false %}\n {% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}\n {% else %}\n {% set content = message['content'] %}\n {% endif %}\n {% if message['role'] == 'user' %}\n {{- bos_token + '[TRANS]\\n' + content + '\\n[/TRANS]\\n' -}}\n {% elif message['role'] == 'system' %}\n {{- '<<SYS>>\\n' + content + '\\n<</SYS>>\\n\\n' -}}\n {% elif message['role'] == 'assistant' %}\n {{- content + eos_token -}}\n {% endif %}\n{% endfor %}\n",
|
41 |
"clean_up_tokenization_spaces": false,
|
42 |
"eos_token": "</s>",
|
43 |
"legacy": false,
|
|
|
37 |
}
|
38 |
},
|
39 |
"bos_token": "<s>",
|
|
|
40 |
"clean_up_tokenization_spaces": false,
|
41 |
"eos_token": "</s>",
|
42 |
"legacy": false,
|