init

config.json

@@ -0,0 +1,40 @@
+{
+  "apply_residual_connection_post_layernorm": false,
+  "architectures": [
+    "TelechatForCausalLM"
+  ],
+  "auto_map": {
+	"AutoConfig": "configuration_telechat.TelechatConfig",
+	"AutoModelForCausalLM": "modeling_telechat.TelechatForCausalLM"
+  },
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_dropout": 0.0,
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-08,
+  "masked_softmax_fusion": true,
+  "model_type": "telechat",
+  "n_head": 64,
+  "n_inner": null,
+  "num_key_value_heads": 8,
+  "n_layer": 96,
+  "pad_token_id": 3,
+  "pretraining_tp": 2,
+  "skip_bias_add": false,
+  "skip_bias_add_qkv": false,
+  "slow_but_exact": false,
+  "unk_token_id": 0,
+  "use_cache": true,
+  "vocab_size": 131072,
+  "ffn_hidden_size": 40960,
+  "flash_attn":true,
+  "tie_word_embeddings":false,
+  "training_seqlen":8192,
+  "base_seqlen":8192,
+  "seq_length": 8192
+}
+

configuration.json

@@ -0,0 +1 @@
+{"task":"text-generation"}

configuration_telechat.py

@@ -0,0 +1,94 @@
+# coding=utf-8
+# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Telechat configuration"""
+
+from packaging import version
+from collections import OrderedDict
+from transformers.utils import is_torch_available, logging
+from transformers.configuration_utils import PretrainedConfig
+from typing import TYPE_CHECKING, Any, List, Mapping, Optional
+
+logger = logging.get_logger(__name__)
+
+class TelechatConfig(PretrainedConfig):
+    """
+    Args:
+        vocab_size (`int`, *optional*, defaults to 160256): Vocabulary size of the Telechat model.
+        hidden_size (`int`, *optional*, defaults to 4096): Dimensionality of the embeddings and hidden states.
+        ffn_hidden_size (`int`, *optional*, defaults to 12288): Dimensionality of the feed-forward hidden states.
+        n_layer (`int`, *optional*, defaults to 30): Number of hidden layers in the Transformer
+        n_head (`int`, *optional*, defaults to 32): Number of attention heads for each attention layer.
+        layer_norm_epsilon (`float`, *optional*, defaults to 1e-5): The epsilon to use in the layer normalization layers.
+        initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        apply_residual_connection_post_layernorm (`bool`, *optional*, defaults to `False`): If enabled, use the layer norm of the hidden states as the residual in the transformer blocks
+        hidden_dropout (`float`, *optional*, defaults to 0.0): Dropout rate of the dropout function on the bias dropout.
+        attention_dropout (`float`, *optional*, defaults to 0.0): Dropout rate applied to the attention probs
+        use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions.
+        training_seqlen (`int`, *optional*, defaults to 8192): Sequence length during last finetuning.
+        logn (`bool`, *optional*, defaults to `True`): Whether or not to use logN during extrapolation.
+        embed_layernorm (`bool`, *optional*, defaults to `True`): Whether or not to use embedding layernorm.
+
+    """
+
+    model_type = "telechat"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {
+        "num_hidden_layers": "n_layer",
+        "num_attention_heads": "n_head",
+    }
+
+    def __init__(
+        self,
+        vocab_size=160256,
+        hidden_size=4096,
+        n_layer=30,
+        n_head=32,
+        layer_norm_epsilon=1e-5,
+        initializer_range=0.02,
+        use_cache=True,
+        bos_token_id=1,
+        eos_token_id=2,
+        apply_residual_connection_post_layernorm=False,
+        hidden_dropout=0.0,
+        attention_dropout=0.0,
+        ffn_hidden_size=12288,
+        training_seqlen = 8192,
+        logn = True,
+        embed_layernorm = False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        n_embed = kwargs.pop("n_embed", None)
+        self.hidden_size = hidden_size if n_embed is None else n_embed
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
+        self.use_cache = use_cache
+        self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm
+        self.hidden_dropout = hidden_dropout
+        self.attention_dropout = attention_dropout
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.logn = logn
+        self.ffn_hidden_size = ffn_hidden_size
+        self.training_seqlen = training_seqlen
+        self.embed_layernorm = embed_layernorm
+        self.num_key_value_heads= kwargs.pop("num_key_value_heads", None)
+
+
+        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "max_length": 8192,
+  "do_sample": false,
+  "use_cache": true,
+  "temperature": 0.3,
+  "top_k": 5,
+  "top_p": 0.85,
+  "repetition_penalty": 1.03,
+  "pad_token_id": 3,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "user_token_id": 4,
+  "bot_token_id": 5
+}

generation_utils.py

@@ -0,0 +1,162 @@
+from typing import Optional
+from collections import deque
+from queue import Queue
+import copy
+
+
+class History:
+
+    def __init__(self, tokenizer, history):
+        '''
+        init from a list of dict
+        '''
+        # use deque to meet some special situation
+        self.input_history = deque()
+        self.tokenizer = tokenizer
+        if history:
+            self._transfer_from_list(history)
+
+    def _transfer_from_list(self, history):
+        for message in history:
+            content = message.get("content")
+            # the token result may not be equal to the result model gen
+            message.update(self.tokenizer(content))
+            self.input_history.append(message)
+
+    def append(self, message):
+        content = message.get("content")
+        if "input_ids" not in message or "attention_mask" not in message:
+            message.update(self.tokenizer(content))
+        self.input_history.append(message)
+
+    def append_left(self, message):
+        content = message.get("content")
+        if "input_ids" not in message or "attention_mask" not in message:
+            message.update(self.tokenizer(content))
+        self.input_history.appendleft(message)
+
+    def pop(self):
+        x = self.input_history.pop()
+        return x
+
+    def pop_left(self):
+        x = self.input_history.pop_left()
+        return x
+
+    def update(self, message):
+        self.input_history.pop()
+        self.append(message)
+
+    def __len__(self):
+        return self.input_history.__len__()
+
+    def __str__(self):
+        return self.input_history.__str__()
+
+    def __copy__(self):
+        new_instance = type(self)(self.tokenizer, [])
+        new_instance.input_history = copy.copy(self.input_history)
+        return new_instance
+
+    def __deepcopy__(self, memodict={}):
+        new_instance = type(self)(self.tokenizer, [])
+        new_instance.input_history = copy.deepcopy(self.input_history)
+        return new_instance
+
+
+class TelechatIterTextStreamer:
+    """
+    With reference to the TextIterStreamers in transformers, we have rewritten this class
+    """
+
+    def __init__(
+            self, tokenizer, history: History = None, skip_prompt: bool = False, timeout: Optional[float] = None,
+            **decode_kwargs
+    ):
+
+        self.tokenizer = tokenizer
+        self.history = history
+        self.skip_prompt = skip_prompt
+        self.timeout = timeout
+        self.decode_kwargs = decode_kwargs
+
+        self.text_queue = Queue()
+        self.cache_time = 0
+        self.text_until = ""
+        self.token_until = []
+        self.stop_signal = None
+        self.next_tokens_are_prompt = True
+
+        self.history.append({"role": "bot", "content": self.text_until})
+
+    def put(self, value):
+        """
+        put printable text into queue
+        """
+        if len(value.shape) > 1 and value.shape[0] > 1:
+            raise ValueError("TextStreamer only supports batch size 1")
+        elif len(value.shape) > 1:
+            value = value[0]
+
+        if self.skip_prompt and self.next_tokens_are_prompt:
+            self.next_tokens_are_prompt = False
+            return
+
+        if value[-1] == self.tokenizer.eos_token_id:
+            return
+
+        # there may be some smart way to decode.
+        self.token_until.extend(value.tolist())
+        text = self.tokenizer.decode(self.token_until, **self.decode_kwargs)
+
+
+        if self._is_printable(text) or self.cache_time >= 6:
+            output_text = text[len(self.text_until):]
+            self.text_until = text
+
+        else:
+            self.cache_time+=1
+            return
+
+        self.on_finalized_text(output_text)
+
+    def end(self):
+        """Flushes any remaining cache and prints a newline to stdout."""
+        # Flush the cache, if it exists
+        text = self.tokenizer.decode(self.token_until, **self.decode_kwargs)
+        output_text = text[len(self.text_until):]
+        self.text_until = text
+        self.on_finalized_text(output_text, stream_end=True)
+        self.clear_cache()
+
+    def clear_cache(self):
+        self.cache_time = 0
+        self.token_until = []
+        self.text_until = ""
+        self.history = None
+        self.next_tokens_are_prompt = True
+
+    def on_finalized_text(self, text: str, stream_end: bool = False):
+        """Put the text tuple in the queue."""
+        self.history.update({"role": "bot", "content": self.text_until, "input_ids": self.token_until,
+                             "attention_mask": [1] * len(self.token_until)})
+        self.text_queue.put((text, self.history), timeout=self.timeout)
+        if stream_end:
+            self.text_queue.put((self.stop_signal, self.history), timeout=self.timeout)
+
+    @staticmethod
+    def _is_printable(cp):
+        """Checks whether tokens can be decoded or not"""
+        if "�" in cp:
+            return False
+        return True
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        value_now, history_until = self.text_queue.get(timeout=self.timeout)
+        if value_now == self.stop_signal:
+            raise StopIteration()
+        else:
+            return value_now, history_until

model-00001-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50a4549e91bf98dcb213f6455f92440454fdd012cd0369e0fdb2f019c83fe4c7
+size 8925611272

model-00002-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:548f67873b1f538bf325590f30dbe267600356ccd1bef73c1f1cc7a2902c2aa7
+size 9261287712

model-00003-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e855d3528914e522dcad5348c2229dc2e88fa55e87416f7d5106437d3679a8d3
+size 9261287712

model-00004-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:509b7084b8dcb308c7c7d22fa988e805b6f6947e356526f60cccdd40ceb2a11d
+size 9261287712

model-00005-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb044a2e88e6e7a0817459f7f409eeb5abd00836346f69f8ebfd9d0c47028ebf
+size 9261287712

model-00006-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3e410d45493b53ba09cdb8d368ecad07df8c1db96a58d928cd7045c8ed7a48f
+size 9261287728

model-00007-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9cdcab0486ef69e5eedbb99b08f0b55c918a83f227194cb8b8937f010c950a7
+size 9261287736

model-00008-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2a7cf122475960568ea086f9b98b2283dbf9344dda2720726a52bff7f31edbc
+size 9261287736

model-00009-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc5d19f0add344ac71fb3855ca8f04e588fd4d5263e05709ec4dd668ebcf8ee4
+size 9261287736

model-00010-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d9763dc2004ea25ca751d271f92a1cce636f92c6991077ea391788428cfc6c1
+size 9261287736

model-00011-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca7e26c5ed4c0fedec09a4cceef1ac7e03d037c921c18d1df0b782ca4fe7b71b
+size 9261287736

model-00012-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0390b0e55acc47cebac2bc81e57e1e7d3f02f72a0cad2b01b5165caf976ab0b
+size 9261287736

model-00013-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbb3ccd22fc048c59e67dab8df54264acf53d7d62fc98fb49ffeafe7ea181739
+size 9261287736

model-00014-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e2c53dae773a55d42d3265d0d85ae02c8f6f70543a90d923823cab68d44d5ce
+size 9261287736

model-00015-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b813a2adab5ff776a06986e60d22e917c596122c417a7d6ae1930b8ed5d61a4f
+size 9261287736

model-00016-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33976aac9f40e0ff430435fe0295ba1f4443dc4b78233c077de95453eff51d01
+size 9261287736

model-00017-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:096850a1c187e804c370956830ef94d20731f79ee1078fa0613604c93015d305
+size 9261287736

model-00018-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7ed47b73f1171ee30de651924ab58aaa7088037b9f96203be23b4b2514dfb27
+size 9261287736

model-00019-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe1e87f96e41804e06c966605e6b00e0a9c6b51c504ee5e22dfdb28e90d1308f
+size 9261287736

model-00020-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93bc200e651f67d4910170de85235af15b3228b23192afb5c32d5126daaaf996
+size 9261287736

model-00021-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:645df22333bb21b8cef574bdadcaecaf0efcf43d3e747a0b0c0d462740d0b7c4
+size 9261287736

model-00022-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a288d0dafad8ca11e21c869a31d62057f6bc13036934e306cb68cd7fdd5861f
+size 9261287736

model-00023-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94e110e13a08711bfd02cbb3d20e32baecd675f2593a8ae66ba1fcf039e34073
+size 9261287736

model-00024-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56894143f2c2a3ff2af49e7f3ec2af9929ff8d95fe7e8e63b709280ba4babe52
+size 9261287736

model-00025-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72d4aee125d0a41ceb2afc164c78d25b9edc89cf845da5f91e9c60b743b2f3a7
+size 9261287736

model-00026-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72fdd61ca5c065b386092a0701f6fe8ed7ae8d99b0eaba87e50e60acff058876
+size 9261287736

model-00027-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:764f995de7385ab9c14e7393ee6adc73b557c1a182efdcbbf143c3fe22f353c3
+size 9261287736

model-00028-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba9bd63f6e97a17e03b467629d9a4bafd581da1b9445b134423c75d5aa409c08
+size 9261287736

model-00029-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21c3f8b212fef1678419b95693121290430930dd2930bd531aa7a3faa40fac41
+size 9261287736

model-00030-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b861e871ac1d1ae14b6fcadaefe876c45d694f715eb91d68bacf114c2c426adf
+size 9261287736

model-00031-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:495ecf892bce401191ce699032b3d95b957d935987ee462df76de2b5928c1127
+size 9261287736

model-00032-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2613dc2c31320d4c300e260aac5a313dc183b8fdd140f6b3aea338667126ffae
+size 9261287736

model-00033-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67348068c452d63d6ea33f468f810157993b1431c823fc4fc68d1e4c7cf1b653
+size 9261287736

model-00034-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e28056f9391f029ddea9743445dbf763c265feb31a184ba73a7e8de91280d981
+size 9261287736

model-00035-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c966308e58af945da780ddf143e6f436f21a8019ddb1824200822ac8ecf72c8
+size 9261287736

model-00036-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:998076e5d150bea2630b24af7bf1da793a951f5c5b9ddfeb7b9400eb1188eea6
+size 9261287736

model-00037-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1facd7663e7db14ec43876cc727f3d8e75b40bb025cb7a25fb7d2f9a60de68f9
+size 9261287736

model-00038-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c06aafaf597e562ebce7672650df8ae3fcabf22977c4d47004f817ca4f8fe536
+size 9261287736

model-00039-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3bda7f14ffa81266bd9ec07e05e9c3bc16726f8fcd75281d4af4002508bfceb
+size 9261287736

model-00040-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf19d0fbf517593390ccc10b1c00c1410b54286ca5c651c1fe3ee8de7b02c4d1
+size 9261287736

model-00041-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f98f5ea063046592be25906b7dc84f0096e67f104f1bcd847858ee778ba4e1eb
+size 9261287736

model-00042-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:033ba66aecb88ed950b1c49bd2c6f6d5e5173ed4882b22ff189f4caef542ec9f
+size 9261287736

model-00043-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a46195067f9f791eee221c2195d30136f0f8eb723dd0c1674d68c6c2409fa68
+size 9261287736

model-00044-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5becd94d0a4e65b2b0496f21ca679ba4b171ffb32851eabd15af17004bda901
+size 9261287736

model-00045-of-00050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:460bd2d228da729c1ef006c65582a8f8d709c748a731f56ff6129ec8beb7cd57
+size 9261287736