{ "tokenizer_class": "Telechat2Tokenizer", "auto_map": { "AutoTokenizer": [ "tokenization_telechat2.Telechat2Tokenizer", null ] }, "added_tokens_decoder": { "1": { "content": "<_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "<_end>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "<_pad>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "<_user>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "<_bot>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<_start>", "<_end>", "<_pad>", "<_user>", "<_bot>" ], "add_bos_token": false, "add_eos_token": false, "use_fast": false, "clean_up_tokenization_spaces": false, "split_special_tokens": false, "model_max_length": 100000000, "sp_model_kwargs": {}, "bos_token": "<_start>", "eos_token": "<_end>", "pad_token": "<_pad>", "chat_template": "<_start>{%- for message in messages %}{% if message.role == 'user' %}{{'<_user>'+message.content}}{% endif %}{% if message.role == 'bot' %}{{'<_bot>'+message.content+'<_end>'}}{% endif %}{% endfor %}{% if add_generation_prompt %}<_bot>{% endif %}" }