Support single integer or empty list as input to decode (#7)
Browse files- Update tokenization_chatglm.py (dffe870a7ef1558ebbc6f3dfdf46491cdb2b3e31)
Co-authored-by: Yichao 'Peak' Ji <[email protected]>
- tokenization_chatglm.py +4 -0
tokenization_chatglm.py
CHANGED
@@ -261,6 +261,10 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
261 |
spaces_between_special_tokens: bool = True,
|
262 |
**kwargs
|
263 |
) -> str:
|
|
|
|
|
|
|
|
|
264 |
if isinstance(token_ids[0], list):
|
265 |
tokens = []
|
266 |
for single_token_ids in token_ids:
|
|
|
261 |
spaces_between_special_tokens: bool = True,
|
262 |
**kwargs
|
263 |
) -> str:
|
264 |
+
if not isinstance(token_ids, list):
|
265 |
+
token_ids = [token_ids]
|
266 |
+
if len(token_ids) == 0:
|
267 |
+
return ""
|
268 |
if isinstance(token_ids[0], list):
|
269 |
tokens = []
|
270 |
for single_token_ids in token_ids:
|