Tokenizer suggestions

README.md

@@ -6,13 +6,7 @@ license: apache-2.0
 
 Official research release for the family of **XGen** models (`7B`) by Salesforce AI Research:
 
-*Title*: [Long Sequence Modeling with XGen: A 7B LLM Trained on 8K Input Sequence Length](https://arxiv.org/abs/2309.03450)
-
-*Authors*: [Erik Nijkamp](https://eriknijkamp.com)\*, Tian Xie\*, [Hiroaki Hayashi](https://hiroakih.me/)\*, [Bo Pang](https://scholar.google.com/citations?user=s9fNEVEAAAAJ&hl=en)\*, Congying Xia\*, Chen Xing, Jesse Vig, Semih Yavuz, Philippe Laban, Ben Krause, Senthil Purushwalkam, Tong Niu, Wojciech Kryscinski, Lidiya Murakhovs'ka, Prafulla Kumar Choubey, Alex Fabbri, Ye Liu, Rui Meng, Lifu Tu, Meghana Bhat, [Chien-Sheng Wu](https://jasonwu0731.github.io/), Silvio Savarese, [Yingbo Zhou](https://scholar.google.com/citations?user=H_6RQ7oAAAAJ&hl=en), [Shafiq Rayhan Joty](https://raihanjoty.github.io/), [Caiming Xiong](http://cmxiong.com/).
-
-(* indicates equal contribution)
-
-Correspondence to: [Shafiq Rayhan Joty](mailto:[email protected]), [Caiming Xiong](mailto:[email protected])
+*Title*: [Long Sequence Modeling with XGen: A 7B LLM Trained on 8K Input Sequence Length](https://blog.salesforceairesearch.com/xgen/)
 
 ## Models
 
@@ -55,9 +49,9 @@ print(tokenizer.decode(sample[0]))
 ```bibtex
 @misc{XGen,
   title={Long Sequence Modeling with XGen: A 7B LLM Trained on 8K Input Sequence Length},
-  author={Erik Nijkamp, Tian Xie, Hiroaki Hayashi, Bo Pang, Congying Xia, Chen Xing, Jesse Vig, Semih Yavuz, Philippe Laban, Ben Krause, Senthil Purushwalkam, Tong Niu, Wojciech Kryscinski, Lidiya Murakhovs'ka, Prafulla Kumar Choubey, Alex Fabbri, Ye Liu, Rui Meng, Lifu Tu, Meghana Bhat, Chien-Sheng Wu, Silvio Savarese, Yingbo Zhou, Shafiq Rayhan Joty, Caiming Xiong},
-  howpublished={ArXiv},
+  author={Salesforce AI Research},
+  howpublished={Salesforce AI Research Blog},
   year={2023},
-  url={https://arxiv.org/abs/2309.03450}
+  url={https://blog.salesforceairesearch.com/xgen-7b/}
 }
 ```

config.json

@@ -2,8 +2,8 @@
   "architectures": [
     "LlamaForCausalLM"
   ],
-  "bos_token_id": 50256,
-  "eos_token_id": 50256,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,

generation_config.json

@@ -1,6 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 50256,
-  "eos_token_id": 50256,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
   "transformers_version": "4.29.2"
 }

tokenization_xgen.py

@@ -25,7 +25,7 @@ MAX_MODEL_INPUT_SIZES = {
 }
 
 
-def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):
+def tiktoken_tokenizer(base="gpt2", add_special=True):
     if not add_special:
         return tiktoken.get_encoding(base)
 
@@ -60,18 +60,9 @@ def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):
         ]
         return fim_tokens
 
-    def include_additional_tokens():
-        tokens = []
-        tokens += [f"<dummy_{i}>" for i in range(4)]
-        tokens.append("<sep>")  # 50317
-        tokens.append("<eom>")  # 50318
-        tokens += [f"<mask_{i}>" for i in reversed(range(1, 51199-50318+1))]
-        return tokens        
-
     add_whitespaces = include_whitespace(n_min=2, n_max=32)
     add_tabs = include_tabs(n_min=2, n_max=10)
     fim_tokens = include_fim_tokens()
-    additional_tokens = include_additional_tokens()
 
     tokenizer = tiktoken.get_encoding(base)
 
@@ -91,13 +82,7 @@ def tiktoken_tokenizer(base="gpt2", pad_token=None, add_special=True):
     for sp in fim_tokens:
         special_tokens[sp] = idx
         idx += 1
-    for sp in additional_tokens:
-        special_tokens[sp] = idx
-        idx += 1        
 
-    if pad_token and pad_token not in tokenizer._special_tokens and pad_token not in special_tokens:
-        special_tokens[pad_token] = idx
-        idx += 1
     # In production, load the arguments directly instead of accessing private attributes
     # See openai_public.py for examples of arguments for specific encodings
     enc = tiktoken.Encoding(
@@ -127,22 +112,19 @@ class XgenTokenizer(PreTrainedTokenizer):
     def __init__(
             self,
             pad_token=None,
-            eos_token="<|endoftext|>",
             add_eos_token=False,
             add_special_tokens=True,
             **kwargs,
     ):
-        pad_token_added = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
-        eos_token_added = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
-        self.add_eos_token = add_eos_token
-        self.encoder = tiktoken_tokenizer(base="gpt2", pad_token=pad_token, add_special=add_special_tokens)
+        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
         super().__init__(
-            pad_token=pad_token_added,
-            eos_token=eos_token_added,
+            pad_token=pad_token,
             add_eos_token=add_eos_token,
             add_special_tokens=add_special_tokens,
             **kwargs,
         )
+        self.add_eos_token = add_eos_token
+        self.encoder = tiktoken_tokenizer(base="gpt2", add_special=add_special_tokens)
 
     @property
     def vocab_size(self):
@@ -151,7 +133,7 @@ class XgenTokenizer(PreTrainedTokenizer):
 
     def get_vocab(self):
         """Returns vocab as a dict"""
-        vocab = {self.encoder.decode_single_token_bytes(i): i for i in range(self.vocab_size)}
+        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
         return vocab
 
     def _tokenize(self, text, **kwargs):
@@ -160,25 +142,18 @@ class XgenTokenizer(PreTrainedTokenizer):
 
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
-        if isinstance(token, str):
-            return self.encoder.encode_single_token(token)
-        else:
-            return token
+        return token
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
-        return self.encoder.decode_single_token_bytes(index).decode("utf-8")
+        return self.encoder.decode_single_token_bytes(index)
 
-    def _decode(self, token_ids, skip_special_tokens: bool = False, **kwargs):
-        if not isinstance(token_ids, list):
-            token_ids = [token_ids]
-        if skip_special_tokens:
-            token_ids = [t for t in token_ids if t not in self.all_special_ids]
+    def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
         return self.encoder.decode(token_ids)
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
         """Build model inputs from a sequence by appending eos_token_id."""
-        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+        eos_token_id = [50256] if self.add_eos_token else []
 
         output = token_ids_0 + eos_token_id
 
@@ -234,7 +209,7 @@ class XgenTokenizer(PreTrainedTokenizer):
         Returns:
             `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
         """
-        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+        eos_token_id = [50256] if self.add_eos_token else []
 
         output = [0] * len(token_ids_0 + eos_token_id)
 
@@ -242,7 +217,3 @@ class XgenTokenizer(PreTrainedTokenizer):
             output += [1] * len(token_ids_1 + eos_token_id)
 
         return output
-
-    # has no vocab file
-    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
-        return ()

tokenizer_config.json

@@ -2,7 +2,6 @@
   "add_eos_token": false,
   "add_special_tokens": true,
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
   "tokenizer_class": "XgenTokenizer",