|
{ |
|
"additional_special_tokens": [ |
|
"<|empty|>", |
|
"<|very_short|>", |
|
"<|short|>", |
|
"<|long|>", |
|
"<|very_long|>", |
|
"<|tag_to_long|>", |
|
"<|long_to_tag|>", |
|
"<|short_to_tag|>", |
|
"<|short_to_long|>", |
|
"<|tag_to_short_to_long|>", |
|
"<|short_to_tag_to_long|>", |
|
"<|short_to_long_to_tag|>", |
|
"<|gen_meta|>" |
|
], |
|
"bos_token": { |
|
"content": "<s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false |
|
}, |
|
"eos_token": { |
|
"content": "</s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false |
|
}, |
|
"pad_token": { |
|
"content": "</s>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false |
|
}, |
|
"unk_token": { |
|
"content": "<unk>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false |
|
} |
|
} |
|
|