Add gmask token id
Browse files- tokenization_chatglm.py +6 -0
tokenization_chatglm.py
CHANGED
@@ -200,6 +200,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
200 |
|
201 |
""" Initialisation """
|
202 |
|
|
|
|
|
|
|
|
|
|
|
|
|
203 |
@property
|
204 |
def eop_token_id(self) -> Optional[int]:
|
205 |
"""
|
|
|
200 |
|
201 |
""" Initialisation """
|
202 |
|
203 |
+
@property
|
204 |
+
def gmask_token_id(self) -> Optional[int]:
|
205 |
+
if self.gmask_token is None:
|
206 |
+
return None
|
207 |
+
return self.convert_tokens_to_ids(self.gmask_token)
|
208 |
+
|
209 |
@property
|
210 |
def eop_token_id(self) -> Optional[int]:
|
211 |
"""
|