liuxz0801 commited on
Commit
168055a
1 Parent(s): 5dbefa9

initialize

config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "../models/7B",
3
+ "apply_residual_connection_post_layernorm": false,
4
+ "architectures": [
5
+ "TelechatForCausalLM"
6
+ ],
7
+ "attention_dropout": 0.0,
8
+ "attention_softmax_in_fp32": true,
9
+ "auto_map": {
10
+ "AutoConfig": "configuration_telechat.TelechatConfig",
11
+ "AutoModelForCausalLM": "modeling_telechat.TelechatForCausalLM"
12
+ },
13
+ "bias_dropout_fusion": true,
14
+ "bos_token_id": 1,
15
+ "embed_layernorm": false,
16
+ "eos_token_id": 2,
17
+ "ffn_hidden_size": 12288,
18
+ "flash_attn": true,
19
+ "hidden_dropout": 0.0,
20
+ "hidden_size": 4096,
21
+ "initializer_range": 0.02,
22
+ "layer_norm_epsilon": 1e-05,
23
+ "logn": false,
24
+ "masked_softmax_fusion": true,
25
+ "model_type": "telechat",
26
+ "n_head": 32,
27
+ "n_inner": null,
28
+ "n_layer": 30,
29
+ "offset_alibi": 100,
30
+ "pad_token_id": 3,
31
+ "pretraining_tp": 2,
32
+ "seq_length": 8192,
33
+ "skip_bias_add": true,
34
+ "skip_bias_add_qkv": false,
35
+ "slow_but_exact": false,
36
+ "torch_dtype": "float16",
37
+ "training_seqlen": 4096,
38
+ "transformers_version": "4.30.0",
39
+ "unk_token_id": 0,
40
+ "use_cache": true,
41
+ "vocab_size": 160256
42
+ }
configuration_telechat.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 the Big Science Workshop and HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """ Telechat configuration"""
17
+
18
+ from transformers.utils import logging
19
+ from transformers.configuration_utils import PretrainedConfig
20
+
21
+ logger = logging.get_logger(__name__)
22
+
23
+ class TelechatConfig(PretrainedConfig):
24
+ """
25
+ Args:
26
+ vocab_size (`int`, *optional*, defaults to 160256): Vocabulary size of the Telechat model.
27
+ hidden_size (`int`, *optional*, defaults to 4096): Dimensionality of the embeddings and hidden states.
28
+ ffn_hidden_size (`int`, *optional*, defaults to 12288): Dimensionality of the feed-forward hidden states.
29
+ n_layer (`int`, *optional*, defaults to 30): Number of hidden layers in the Transformer
30
+ n_head (`int`, *optional*, defaults to 32): Number of attention heads for each attention layer.
31
+ layer_norm_epsilon (`float`, *optional*, defaults to 1e-5): The epsilon to use in the layer normalization layers.
32
+ initializer_range (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
33
+ apply_residual_connection_post_layernorm (`bool`, *optional*, defaults to `False`): If enabled, use the layer norm of the hidden states as the residual in the transformer blocks
34
+ hidden_dropout (`float`, *optional*, defaults to 0.0): Dropout rate of the dropout function on the bias dropout.
35
+ attention_dropout (`float`, *optional*, defaults to 0.0): Dropout rate applied to the attention probs
36
+ use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions.
37
+ training_seqlen (`int`, *optional*, defaults to 8192): Sequence length during last finetuning.
38
+ logn (`bool`, *optional*, defaults to `True`): Whether or not to use logN during extrapolation.
39
+ embed_layernorm (`bool`, *optional*, defaults to `True`): Whether or not to use embedding layernorm.
40
+
41
+ """
42
+
43
+ model_type = "telechat"
44
+ keys_to_ignore_at_inference = ["past_key_values"]
45
+ attribute_map = {
46
+ "num_hidden_layers": "n_layer",
47
+ "num_attention_heads": "n_head",
48
+ }
49
+
50
+ def __init__(
51
+ self,
52
+ vocab_size=160256,
53
+ hidden_size=4096,
54
+ n_layer=30,
55
+ n_head=32,
56
+ layer_norm_epsilon=1e-5,
57
+ initializer_range=0.02,
58
+ use_cache=True,
59
+ bos_token_id=1,
60
+ eos_token_id=2,
61
+ apply_residual_connection_post_layernorm=False,
62
+ hidden_dropout=0.0,
63
+ attention_dropout=0.0,
64
+ ffn_hidden_size=12288,
65
+ training_seqlen = 8192,
66
+ logn = True,
67
+ embed_layernorm = False,
68
+ **kwargs,
69
+ ):
70
+ self.vocab_size = vocab_size
71
+ n_embed = kwargs.pop("n_embed", None)
72
+ self.hidden_size = hidden_size if n_embed is None else n_embed
73
+ self.n_layer = n_layer
74
+ self.n_head = n_head
75
+ self.layer_norm_epsilon = layer_norm_epsilon
76
+ self.initializer_range = initializer_range
77
+ self.use_cache = use_cache
78
+ self.apply_residual_connection_post_layernorm = apply_residual_connection_post_layernorm
79
+ self.hidden_dropout = hidden_dropout
80
+ self.attention_dropout = attention_dropout
81
+ self.bos_token_id = bos_token_id
82
+ self.eos_token_id = eos_token_id
83
+ self.logn = logn
84
+ self.ffn_hidden_size = ffn_hidden_size
85
+ self.training_seqlen = training_seqlen
86
+ self.embed_layernorm = embed_layernorm
87
+
88
+
89
+ super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
90
+
generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "max_length": 4096,
3
+ "do_sample": false,
4
+ "use_cache": true,
5
+ "temperature": 0.3,
6
+ "top_k": 5,
7
+ "top_p": 0.85,
8
+ "repetition_penalty": 1.03,
9
+ "pad_token_id": 3,
10
+ "bos_token_id": 160132,
11
+ "eos_token_id": 160133,
12
+ "user_token_id": 160130,
13
+ "bot_token_id": 160131
14
+ }
generation_utils.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+ from collections import deque
3
+ from queue import Queue
4
+ import copy
5
+
6
+
7
+ class History:
8
+
9
+ def __init__(self, tokenizer, history):
10
+ '''
11
+ init from a list of dict
12
+ '''
13
+ # use deque to meet some special situation
14
+ self.input_history = deque()
15
+ self.tokenizer = tokenizer
16
+ if history:
17
+ self._transfer_from_list(history)
18
+
19
+ def _transfer_from_list(self, history):
20
+ for message in history:
21
+ content = message.get("content")
22
+ message.update(self.tokenizer(content))
23
+ self.input_history.append(message)
24
+
25
+ def append(self, message):
26
+ content = message.get("content")
27
+ message.update(self.tokenizer(content))
28
+ self.input_history.append(message)
29
+
30
+ def append_left(self, message):
31
+ content = message.get("content")
32
+ message.update(self.tokenizer(content))
33
+ self.input_history.appendleft(message)
34
+
35
+ def pop(self):
36
+ x = self.input_history.pop()
37
+ return x
38
+
39
+ def pop_left(self):
40
+ x = self.pop_left()
41
+ return x
42
+
43
+ def update(self, content: str):
44
+ x = self.input_history.pop()
45
+ self.append({"role": x["role"], "content": content})
46
+
47
+ def __len__(self):
48
+ return self.input_history.__len__()
49
+
50
+ def __str__(self):
51
+ return self.input_history.__str__()
52
+
53
+ def __copy__(self):
54
+ new_instance = type(self)(self.tokenizer, [])
55
+ new_instance.input_history = copy.copy(self.input_history)
56
+ return new_instance
57
+
58
+ def __deepcopy__(self, memodict={}):
59
+ new_instance = type(self)(self.tokenizer, [])
60
+ new_instance.input_history = copy.deepcopy(self.input_history)
61
+ return new_instance
62
+
63
+
64
+ class TelechatIterTextStreamer:
65
+ """
66
+ With reference to the TextIterStreamers in transformers, we have rewritten this class
67
+ """
68
+
69
+ def __init__(
70
+ self, tokenizer, history: History = None, skip_prompt: bool = False, timeout: Optional[float] = None,
71
+ **decode_kwargs
72
+ ):
73
+
74
+ self.tokenizer = tokenizer
75
+ self.history = history
76
+ self.skip_prompt = skip_prompt
77
+ self.timeout = timeout
78
+ self.decode_kwargs = decode_kwargs
79
+
80
+ self.text_queue = Queue()
81
+ self.token_cache = []
82
+ self.cache_time = 0
83
+ self.text_until = ""
84
+ self.stop_signal = None
85
+ self.next_tokens_are_prompt = True
86
+
87
+ self.history.append({"role": "bot", "content": self.text_until})
88
+
89
+ def put(self, value):
90
+ """
91
+ put printable text into queue
92
+ """
93
+ if len(value.shape) > 1 and value.shape[0] > 1:
94
+ raise ValueError("TextStreamer only supports batch size 1")
95
+ elif len(value.shape) > 1:
96
+ value = value[0]
97
+
98
+ if self.skip_prompt and self.next_tokens_are_prompt:
99
+ self.next_tokens_are_prompt = False
100
+ return
101
+
102
+ if value[-1] == self.tokenizer.eos_token_id:
103
+ return
104
+
105
+ # there may be some smart way to decode.
106
+ self.token_cache.extend(value.tolist())
107
+ text = self.tokenizer.decode(self.token_cache, **self.decode_kwargs)
108
+ self.cache_time += 1
109
+
110
+ if self._is_printable(text) or self.cache_time >= 6:
111
+ self.text_until += text
112
+ self.token_cache = []
113
+ self.cache_time = 0
114
+
115
+ else:
116
+ return
117
+
118
+ self.on_finalized_text(text)
119
+
120
+ def end(self):
121
+ """Flushes any remaining cache and prints a newline to stdout."""
122
+ # Flush the cache, if it exists
123
+ text = ""
124
+ if len(self.token_cache) > 0:
125
+ text = self.tokenizer.decode(self.token_cache, **self.decode_kwargs)
126
+ self.text_until += text
127
+ self.on_finalized_text(text, stream_end=True)
128
+ self.clear_cache()
129
+
130
+ def clear_cache(self):
131
+ self.cache_time = 0
132
+ self.token_cache = []
133
+ self.text_until = ""
134
+ self.history = None
135
+ self.next_tokens_are_prompt = True
136
+
137
+ def on_finalized_text(self, text: str, stream_end: bool = False):
138
+ """Put the text tuple in the queue."""
139
+ self.history.update(self.text_until)
140
+ self.text_queue.put((text, self.history), timeout=self.timeout)
141
+ if stream_end:
142
+ self.text_queue.put((self.stop_signal, self.history), timeout=self.timeout)
143
+
144
+ @staticmethod
145
+ def _is_printable(cp):
146
+ """Checks whether tokens can be decoded or not"""
147
+ if "�" in cp:
148
+ return False
149
+ return True
150
+
151
+ def __iter__(self):
152
+ return self
153
+
154
+ def __next__(self):
155
+ value_now, history_until = self.text_queue.get(timeout=self.timeout)
156
+ if value_now == self.stop_signal:
157
+ raise StopIteration()
158
+ else:
159
+ return value_now, history_until
gptq_model-8bit-128g.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9db49bdca43688048a3e445770798a802480a3748354bdf47a4bfa3db52ff43
3
+ size 8016770325
modeling_telechat.py ADDED
@@ -0,0 +1,917 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2022 HuggingFace Inc. team and BigScience workshop.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
17
+
18
+ # Copyright (c) 2021 EleutherAI
19
+ # This file is based on code by the authors denoted below and has been modified from its original version.
20
+ #
21
+ # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
22
+ #
23
+ # Licensed under the Apache License, Version 2.0 (the "License");
24
+ # you may not use this file except in compliance with the License.
25
+ # You may obtain a copy of the License at
26
+ #
27
+ # http://www.apache.org/licenses/LICENSE-2.0
28
+ #
29
+ # Unless required by applicable law or agreed to in writing, software
30
+ # distributed under the License is distributed on an "AS IS" BASIS,
31
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
32
+ # See the License for the specific language governing permissions and
33
+ # limitations under the License.
34
+
35
+
36
+ """PyTorch TELECHAT model."""
37
+
38
+ import warnings
39
+ from typing import Optional, Tuple, Union, List, Dict
40
+ from threading import Thread
41
+
42
+ import torch
43
+ import math
44
+ import copy
45
+ from torch import nn
46
+ import torch.utils.checkpoint
47
+ from torch.nn import functional as F
48
+ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss
49
+ from transformers.modeling_outputs import (
50
+ BaseModelOutputWithPastAndCrossAttentions,
51
+ CausalLMOutputWithCrossAttentions
52
+ )
53
+ from transformers.modeling_utils import PreTrainedModel
54
+ from transformers.utils import logging
55
+ from transformers import GenerationConfig
56
+
57
+ from .configuration_telechat import TelechatConfig
58
+ from .generation_utils import History, TelechatIterTextStreamer
59
+
60
+ logger = logging.get_logger(__name__)
61
+
62
+ _CHECKPOINT_FOR_DOC = "telechat"
63
+ _CONFIG_FOR_DOC = "TelechatConfig"
64
+
65
+ TELECHAT_PRETRAINED_MODEL_ARCHIVE_LIST = []
66
+
67
+ try:
68
+ from einops import rearrange
69
+ except ImportError:
70
+ rearrange = None
71
+
72
+ use_flash_attn = True
73
+ try:
74
+ from flash_attn.flash_attn_interface import flash_attn_unpadded_func
75
+ except ImportError:
76
+ try:
77
+ from flash_attn.flash_attn_interface import flash_attn_varlen_func as flash_attn_unpadded_func
78
+ except ImportError:
79
+ flash_attn_unpadded_func = None
80
+
81
+
82
+ class RotaryEmbedding(torch.nn.Module):
83
+ # Extracted from: https://github.com/EleutherAI/gpt-neox
84
+ def __init__(self, dim, config, base=10000, precision=torch.half):
85
+ super().__init__()
86
+ self.config = config
87
+ self.dim = dim
88
+ self.base = base
89
+ self.inv_freq = 1. / (base ** (torch.arange(0, dim, 2).float().half() / dim)).cuda()
90
+ self.max_seq_len_cached = None
91
+ self.cos_cached = None
92
+ self.sin_cached = None
93
+ self.precision = precision
94
+
95
+ def get_mscale(self, scale=1):
96
+ if scale <= 1:
97
+ return 1.0
98
+ return 0.1 * math.log(scale) + 1.0
99
+
100
+ def get_ntk_alpha(self, true_seq_len):
101
+ context_value = math.log(true_seq_len / 4096, 2) + 1
102
+ # ntk_alpha = 2 ** context_value - 1
103
+ ntk_alpha = 2 ** math.ceil(context_value) - 1
104
+ ntk_alpha = max(ntk_alpha, 1)
105
+ return ntk_alpha
106
+
107
+ def forward(self, x, seq_dim=0, seq_len=None):
108
+ if seq_len is None:
109
+ seq_len = x.shape[seq_dim]
110
+ seq_len = max(seq_len, self.config.training_seqlen)
111
+ ntk_alpha = self.get_ntk_alpha(seq_len)
112
+ self.mscale = float(self.get_mscale(seq_len / self.config.training_seqlen))
113
+ if True:
114
+ base = self.base * ntk_alpha ** (self.dim / (self.dim - 2))
115
+ self.inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2, device=x.device).float() / self.dim))
116
+ self.max_seq_len_cached = seq_len
117
+ t = torch.arange(self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype)
118
+ freqs = torch.einsum('i,j->ij', t, self.inv_freq)
119
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
120
+ emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
121
+ if self.precision == torch.bfloat16:
122
+ emb = emb.float()
123
+ # [sx, 1 (b * np), hn]
124
+ self.cos_cached = self.mscale * emb.cos()[:, None, :].half()
125
+ self.sin_cached = self.mscale * emb.sin()[:, None, :].half()
126
+ if self.precision == torch.bfloat16:
127
+ self.cos_cached = self.cos_cached.bfloat16()
128
+ self.sin_cached = self.sin_cached.bfloat16()
129
+ return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]
130
+
131
+
132
+ # rotary pos emb helpers:
133
+ def rotate_half(x):
134
+ x1, x2 = x[..., :x.shape[-1] // 2], x[..., x.shape[-1] // 2:]
135
+ return torch.cat((-x2, x1), dim=x1.ndim - 1) # dim=-1 triggers a bug in earlier torch versions
136
+
137
+
138
+ def apply_rotary_pos_emb_torch(q, k, cos, sin, offset: int = 0): # jitting fails with bf16
139
+ cos, sin = cos[offset:q.shape[0] + offset, ...], sin[offset:q.shape[0] + offset, ...]
140
+ return (q * cos) + (rotate_half(q) * sin), (k * cos) + (rotate_half(k) * sin)
141
+
142
+
143
+ class MixedFusedRMSNorm(nn.Module):
144
+ # Extracted from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py
145
+ def __init__(self, hidden_size, eps=1e-6):
146
+ super().__init__()
147
+ self.weight = nn.Parameter(torch.ones(hidden_size))
148
+ self.variance_epsilon = eps
149
+
150
+ def forward(self, hidden_states):
151
+ input_dtype = hidden_states.dtype
152
+ hidden_states = hidden_states.to(torch.float32)
153
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
154
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
155
+ return self.weight * hidden_states.to(input_dtype)
156
+
157
+
158
+ class FlashSelfAttention(torch.nn.Module):
159
+ # Extracted from https://github.com/microsoft/Megatron-DeepSpeed/blob/main/megatron/model/transformer.py
160
+ """Implement the scaled dot product attention with softmax.
161
+ Arguments
162
+ ---------
163
+ softmax_scale: The temperature to use for the softmax attention.
164
+ (default: 1/sqrt(d_keys) where d_keys is computed at
165
+ runtime)
166
+ attention_dropout: The dropout rate to apply to the attention
167
+ (default: 0.0)
168
+ """
169
+
170
+ def __init__(self, causal=False, softmax_scale=None, attention_dropout=0.0,
171
+ device=None, dtype=None):
172
+ super().__init__()
173
+ assert flash_attn_unpadded_func is not None, ('Please install FlashAttention first, '
174
+ 'e.g., with pip install flash-attn')
175
+ assert rearrange is not None, 'Please install einops first, e.g., with pip install einops'
176
+ self.causal = causal
177
+ self.softmax_scale = softmax_scale
178
+ self.dropout_p = attention_dropout
179
+
180
+ def forward(self, q, k, v):
181
+ """Implements the multihead softmax attention.
182
+ Arguments
183
+ ---------
184
+ q, k, v: The tensor containing the query, key, and value. (B, S, H, D)
185
+ """
186
+ assert all((i.dtype in [torch.float16, torch.bfloat16] for i in (q, k, v)))
187
+ assert all((i.is_cuda for i in (q, k, v)))
188
+
189
+ batch_size, seqlen_q = q.shape[0], q.shape[1]
190
+ seqlen_k = k.shape[1]
191
+
192
+ q, k, v = [rearrange(x, 'b s ... -> (b s) ...') for x in [q, k, v]]
193
+ cu_seqlens_q = torch.arange(0, (batch_size + 1) * seqlen_q, step=seqlen_q, dtype=torch.int32,
194
+ device=q.device)
195
+ self.training = False
196
+ if self.training:
197
+ # during training q,k,v always have same seqlen
198
+ assert seqlen_k == seqlen_q
199
+
200
+ is_causal = self.causal
201
+ cu_seqlens_k = cu_seqlens_q
202
+ dropout_p = self.dropout_p
203
+ else:
204
+ # turn off FA causal mask after first inference autoregressive iteration
205
+ # only on first autoregressive step q,k,v have same seqlen
206
+ is_causal = seqlen_q == seqlen_k
207
+ cu_seqlens_k = torch.arange(0, (batch_size + 1) * seqlen_k, step=seqlen_k, dtype=torch.int32,
208
+ device=q.device)
209
+ dropout_p = 0
210
+
211
+ output = flash_attn_unpadded_func(
212
+ q, k, v, cu_seqlens_q, cu_seqlens_k, seqlen_q, seqlen_k,
213
+ dropout_p=dropout_p,
214
+ softmax_scale=self.softmax_scale, causal=is_causal
215
+ )
216
+
217
+ output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
218
+ return output
219
+
220
+
221
+ def _make_causal_mask(
222
+ input_ids_shape: torch.Size, device: torch.device, past_key_values_length: int
223
+ ) -> torch.BoolTensor:
224
+ """
225
+ Make causal mask used for self-attention.
226
+ """
227
+ batch_size, target_length = input_ids_shape
228
+ mask = torch.empty((target_length, target_length + past_key_values_length), dtype=torch.bool, device=device)
229
+ # ONNX doesn't support `torch.Tensor.triu` properly, thus we use this workaround
230
+ seq_ids = torch.arange(target_length, device=device)
231
+ mask[:, past_key_values_length:] = seq_ids[:, None] < seq_ids[None, :]
232
+
233
+ if past_key_values_length > 0:
234
+ mask[:, :past_key_values_length] = False
235
+
236
+ expanded_mask = mask[None, None, :, :].expand(batch_size, 1, target_length, target_length + past_key_values_length)
237
+ return expanded_mask
238
+
239
+
240
+ def _expand_mask(mask: torch.Tensor, tgt_length: int) -> torch.BoolTensor:
241
+ """
242
+ Expands attention_mask from `[batch_size, src_length]` to `[batch_size, 1, tgt_length, src_length]`.
243
+ """
244
+ batch_size, src_length = mask.shape
245
+ tgt_length = tgt_length if tgt_length is not None else src_length
246
+
247
+ expanded_mask = ~(mask[:, None, None, :].to(torch.bool))
248
+ return expanded_mask.expand(batch_size, 1, tgt_length, src_length)
249
+
250
+
251
+ def dropout_add(x: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor:
252
+ """
253
+ Dropout add function
254
+
255
+ Args:
256
+ x (`torch.tensor`, *required*):
257
+ input tensor
258
+ residual (`torch.tensor`, *required*):
259
+ residual tensor
260
+ prob (`float`, *required*):
261
+ dropout probability
262
+ training (`bool`, *required*):
263
+ training mode
264
+ """
265
+ out = F.dropout(x, p=prob, training=training)
266
+ out = residual + out
267
+ return out
268
+
269
+
270
+ def telechat_gelu_forward(x: torch.Tensor) -> torch.Tensor:
271
+ """
272
+ Custom bias GELU function. Adapted from Megatron-DeepSpeed code. Here we use a simple implementation (inference) to
273
+ make the model jitable.
274
+
275
+ Args:
276
+ x (`torch.tensor`, *required*):
277
+ input hidden states
278
+ """
279
+ return x * 0.5 * (1.0 + torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x)))
280
+
281
+
282
+ def telechat_gelu_back(g: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
283
+ """
284
+ gradient of tanh approximation of gelu gradient of actual gelu is: 0.5 * (1. + torch.erf(x * 0.70710678)) +
285
+ 0.3989423 * x * torch.exp(-0.5 * x * x)
286
+
287
+ Args:
288
+ g (`torch.tensor`, *required*):
289
+ gradient output tensor
290
+ x (`torch.tensor`, *required*):
291
+ input tensor
292
+ """
293
+ x = x[0] # x is a tuple of 1 element, needs to unpack it first
294
+ tanh_out = torch.tanh(0.79788456 * x * (1 + 0.044715 * x * x))
295
+ # sqrt(2/pi) * 3 * 0.044715 -> 0.1070322243
296
+ ff = 0.5 * x * ((1 - tanh_out * tanh_out) * (0.79788456 + 0.1070322243 * x * x)) + 0.5 * (1 + tanh_out)
297
+ return ff * g
298
+
299
+
300
+ class GeLUFunction(torch.autograd.Function):
301
+ @staticmethod
302
+ def forward(ctx, input: torch.Tensor) -> torch.Tensor:
303
+ ctx.save_for_backward(input)
304
+ return telechat_gelu_forward(input)
305
+
306
+ @staticmethod
307
+ def backward(ctx, grad_output: torch.Tensor) -> torch.Tensor:
308
+ input = ctx.saved_tensors
309
+ tmp = telechat_gelu_back(grad_output, input)
310
+ return tmp
311
+
312
+
313
+ class TelechatGelu(nn.Module):
314
+ """
315
+ TelechatBiasGelu wrapper function that make use of the simple function on inference mode to make the model
316
+ torchscriptable and use the autograd function in training mode to get the accurate results of the gradients Partly
317
+ copied from Megatron-DeepSpeed code and adapted for our needs
318
+
319
+ See here why autograd functions are not torchscriptable: https://github.com/pytorch/pytorch/issues/22329
320
+ """
321
+
322
+ def __init__(self):
323
+ super().__init__()
324
+
325
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
326
+ if self.training:
327
+ return GeLUFunction.apply(x)
328
+ else:
329
+ return telechat_gelu_forward(x)
330
+
331
+
332
+ class TelechatAttention(nn.Module):
333
+ def __init__(self, config: TelechatConfig, layer_idx):
334
+ super().__init__()
335
+ self.kv_cache = None
336
+ self.layer_idx = layer_idx
337
+
338
+ self.hidden_size = config.hidden_size
339
+ self.num_heads = config.n_head
340
+ self.head_dim = self.hidden_size // self.num_heads
341
+ self.split_size = self.hidden_size
342
+ self.hidden_dropout = config.hidden_dropout
343
+ self.config = config
344
+
345
+ if self.head_dim * self.num_heads != self.hidden_size:
346
+ raise ValueError(
347
+ f"`hidden_size` must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`:"
348
+ f" {self.num_heads})."
349
+ )
350
+
351
+ # Layer-wise attention scaling
352
+ self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim)
353
+ self.beta = 1.0
354
+
355
+ self.num_key_value_heads = self.num_heads
356
+ kv_projection_size = self.head_dim * self.num_key_value_heads
357
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
358
+ self.query = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
359
+ self.key_value = nn.Linear(self.hidden_size, kv_projection_size * 2, bias=False)
360
+ self.dense = nn.Linear(self.hidden_size, self.hidden_size)
361
+ self.attention_dropout = nn.Dropout(config.attention_dropout)
362
+ self.rotary_emb = RotaryEmbedding(self.head_dim, config=config)
363
+
364
+ self.core_attention_flash = FlashSelfAttention(
365
+ causal=True, attention_dropout=config.attention_dropout
366
+ )
367
+
368
+ self.last_key_layer = None
369
+
370
+ def repeat_kv(self, hidden_states, n_rep):
371
+ slen, batch, num_key_value_heads_per_partition, head_dim = hidden_states.shape
372
+ if n_rep == 1:
373
+ return hidden_states
374
+ hidden_states = hidden_states[:, :, :, None, :].expand(slen, batch, num_key_value_heads_per_partition, n_rep,
375
+ head_dim)
376
+ return hidden_states.reshape(slen, batch, num_key_value_heads_per_partition * n_rep, head_dim)
377
+
378
+ def split_tensor_along_last_dim(self,
379
+ tensor: torch.Tensor,
380
+ num_partitions: int,
381
+ contiguous_split_chunks: bool = False,
382
+ ):
383
+
384
+ # Get the size and dimension.
385
+ last_dim = tensor.dim() - 1
386
+ last_dim_size = tensor.size()[last_dim] // num_partitions
387
+ # Split.
388
+ tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
389
+ # Note: torch.split does not create contiguous tensors by default.
390
+ if contiguous_split_chunks:
391
+ return tuple(chunk.contiguous() for chunk in tensor_list)
392
+
393
+ return tensor_list
394
+
395
+ def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
396
+ batch_size_and_num_heads, seq_length, _ = x.shape
397
+ batch_size = batch_size_and_num_heads // self.num_heads
398
+ x = x.view(batch_size, self.num_heads, seq_length, self.head_dim)
399
+ x = x.permute(0, 2, 1, 3)
400
+ return x.reshape(batch_size, seq_length, self.num_heads * self.head_dim)
401
+
402
+ def forward(
403
+ self,
404
+ hidden_states: torch.Tensor,
405
+ residual: torch.Tensor,
406
+ attention_mask: torch.Tensor,
407
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
408
+ use_cache: bool = False,
409
+ output_attentions: bool = False,
410
+ ):
411
+ hidden_states = hidden_states.transpose(1, 0)
412
+ query_layer = self.query(hidden_states)
413
+ new_tensor_shape = query_layer.size()[:-1] + \
414
+ (self.num_heads,
415
+ self.head_dim)
416
+ query_layer = query_layer.view(*new_tensor_shape)
417
+
418
+ mixed_kv_layer = self.key_value(hidden_states)
419
+ new_tensor_shape = mixed_kv_layer.size()[:-1] + \
420
+ (self.num_key_value_heads,
421
+ 2 * self.head_dim)
422
+ mixed_kv_layer = mixed_kv_layer.view(*new_tensor_shape)
423
+ (key_layer, value_layer) = self.split_tensor_along_last_dim(mixed_kv_layer, 2)
424
+
425
+ output_size = (query_layer.size(1),
426
+ query_layer.size(2),
427
+ query_layer.size(0),
428
+ key_layer.size(0))
429
+
430
+ query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1)
431
+ key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1)
432
+
433
+ apply_rotary_fn = apply_rotary_pos_emb_torch
434
+
435
+ seq_len = key_layer.shape[0]
436
+ offset = 0
437
+
438
+ if use_cache and layer_past != None:
439
+ past_key, past_value = layer_past
440
+ offset = past_key.shape[0]
441
+ seq_len += offset
442
+
443
+ cos, sin = self.rotary_emb(value_layer, seq_len=seq_len)
444
+
445
+ query_layer, key_layer = apply_rotary_fn(query_layer, key_layer, cos, sin, offset=offset)
446
+ if use_cache:
447
+ if layer_past != None:
448
+ past_key, past_value = layer_past
449
+ key_layer = torch.cat((past_key, key_layer[-1, ...].unsqueeze(0)), dim=0)
450
+ value_layer = torch.cat((past_value, value_layer[-1, ...].unsqueeze(0)), dim=0)
451
+ layer_past = key_layer, value_layer
452
+ s, bz, head, dim = value_layer.shape
453
+ s_key = key_layer.shape[0]
454
+ s_query = query_layer.shape[0]
455
+ query_layer = query_layer.reshape((s_query, bz, head, dim))
456
+ key_layer = key_layer.reshape((s_key, bz, head, dim))
457
+
458
+ if self.config.flash_attn:
459
+ q, k, v = [rearrange(x, 's b ... -> b s ...').contiguous() for x in
460
+ (query_layer, key_layer, value_layer)]
461
+ context_layer = self.core_attention_flash(q, k, v)
462
+ context_layer = rearrange(context_layer, 'b s h d -> b s (h d)').contiguous()
463
+ else:
464
+ ##[sq, b, np, hn] -> [sq, b * np, hn]
465
+ query_layer = query_layer.reshape(s_query, bz * self.num_heads, dim)
466
+ # [sk, b, np, hn] -> [sk, b * np, hn]
467
+ key_layer = key_layer.reshape(s_key, bz * self.num_heads, dim)
468
+ matmul_result = self.inv_norm_factor * torch.einsum('bik,bkj->bij', query_layer.transpose(0, 1),
469
+ key_layer.transpose(0, 1).transpose(1, 2))
470
+
471
+ attention_scores = matmul_result.view(bz, self.num_heads, s_query, s_key)
472
+
473
+ input_dtype = attention_scores.dtype
474
+ if input_dtype == torch.float16:
475
+ attention_scores = attention_scores.to(torch.float)
476
+ attn_weights = torch.masked_fill(attention_scores, attention_mask, torch.finfo(attention_scores.dtype).min)
477
+ attention_probs = F.softmax(attn_weights, dim=-1).to(input_dtype) ##dtype = torch.float32
478
+ attention_probs = self.attention_dropout(attention_probs)
479
+ attention_probs_reshaped = attention_probs.view(bz * self.num_heads, s_query, s_key)
480
+
481
+ value_layer = value_layer.reshape(s_key, bz * self.num_heads, dim)
482
+ context_layer = torch.bmm(attention_probs_reshaped, value_layer.transpose(0, 1))
483
+ context_layer = self._merge_heads(context_layer)
484
+
485
+ output_tensor = self.dense(context_layer)
486
+
487
+ output_tensor = dropout_add(output_tensor, residual, self.hidden_dropout, self.training)
488
+ present = None
489
+ outputs = (output_tensor, present)
490
+ if output_attentions:
491
+ outputs += (attention_probs,)
492
+
493
+ return output_tensor, layer_past
494
+
495
+
496
+ class TelechatMLP(nn.Module):
497
+ def __init__(self, config: TelechatConfig):
498
+ super().__init__()
499
+ hidden_size = config.hidden_size
500
+ self.gate_proj = nn.Linear(hidden_size, config.ffn_hidden_size, bias=False)
501
+ self.up_proj = nn.Linear(hidden_size, config.ffn_hidden_size, bias=False)
502
+ self.down_proj = nn.Linear(config.ffn_hidden_size, hidden_size, bias=True)
503
+ self.hidden_dropout = config.hidden_dropout
504
+
505
+ def forward(self, hidden_states: torch.Tensor, residual: torch.Tensor) -> torch.Tensor:
506
+ intermediate_output = self.down_proj(F.silu(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
507
+ output = dropout_add(intermediate_output, residual, self.hidden_dropout, self.training)
508
+ return output
509
+
510
+
511
+ class TelechatBlock(nn.Module):
512
+ def __init__(self, config: TelechatConfig, layer_idx):
513
+ super().__init__()
514
+ hidden_size = config.hidden_size
515
+
516
+ self.input_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon)
517
+ self.num_heads = config.n_head
518
+ self.layer_idx = layer_idx
519
+ self.self_attention = TelechatAttention(config, layer_idx)
520
+ self.post_attention_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon)
521
+
522
+ self.mlp = TelechatMLP(config)
523
+
524
+ self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm
525
+ self.hidden_dropout = config.hidden_dropout
526
+
527
+ def forward(
528
+ self,
529
+ hidden_states: torch.Tensor,
530
+ attention_mask: torch.Tensor,
531
+ layer_past: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
532
+ use_cache: bool = False,
533
+ output_attentions: bool = False,
534
+ ):
535
+ layernorm_output = self.input_layernorm(hidden_states)
536
+ if self.apply_residual_connection_post_layernorm:
537
+ residual = layernorm_output
538
+ else:
539
+ residual = hidden_states
540
+
541
+ attn_outputs = self.self_attention(
542
+ layernorm_output,
543
+ residual,
544
+ layer_past=layer_past,
545
+ attention_mask=attention_mask,
546
+ use_cache=use_cache,
547
+ output_attentions=output_attentions,
548
+ )
549
+
550
+ attention_output = attn_outputs[0]
551
+ outputs = attn_outputs[1:]
552
+ layernorm_output = self.post_attention_layernorm(attention_output)
553
+
554
+ if self.apply_residual_connection_post_layernorm:
555
+ residual = layernorm_output
556
+ else:
557
+ residual = attention_output
558
+ output = self.mlp(layernorm_output, residual)
559
+
560
+ if use_cache:
561
+ outputs = (output,) + outputs
562
+ else:
563
+ outputs = (output,) + outputs[1:]
564
+
565
+ return outputs
566
+
567
+
568
+ class TelechatPreTrainedModel(PreTrainedModel):
569
+ config_class = TelechatConfig
570
+ base_model_prefix = "transformer"
571
+ supports_gradient_checkpointing = True
572
+ _no_split_modules = ["TelechatBlock"]
573
+ _skip_keys_device_placement = "past_key_values"
574
+
575
+ def __init__(self, *inputs, **kwargs):
576
+ super().__init__(*inputs, **kwargs)
577
+
578
+ def _init_weights(self, module: nn.Module):
579
+ """Initialize the weights."""
580
+ if isinstance(module, nn.Linear):
581
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
582
+ if module.bias is not None:
583
+ module.bias.data.zero_()
584
+
585
+ elif isinstance(module, nn.Embedding):
586
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
587
+ if module.padding_idx is not None:
588
+ module.weight.data[module.padding_idx].zero_()
589
+
590
+ elif isinstance(module, LayerNorm):
591
+ module.bias.data.zero_()
592
+ module.weight.data.fill_(1.0)
593
+
594
+ def _set_gradient_checkpointing(self, module: nn.Module, value: bool = False):
595
+ if isinstance(module, TelechatModel):
596
+ module.gradient_checkpointing = value
597
+
598
+
599
+ class TelechatModel(TelechatPreTrainedModel):
600
+ def __init__(self, config: TelechatConfig):
601
+ super().__init__(config)
602
+
603
+ self.embed_dim = config.hidden_size
604
+ self.num_heads = config.n_head
605
+ self.config = config
606
+ self.word_embeddings = nn.Embedding(config.vocab_size, self.embed_dim)
607
+ if self.config.embed_layernorm:
608
+ self.word_embeddings_layernorm = MixedFusedRMSNorm(self.embed_dim, eps=config.layer_norm_epsilon)
609
+
610
+ self.h = nn.ModuleList([TelechatBlock(config, _) for _ in range(config.num_hidden_layers)])
611
+ self.ln_f = MixedFusedRMSNorm(self.embed_dim, eps=config.layer_norm_epsilon)
612
+ self.gradient_checkpointing = False
613
+ self.post_init()
614
+
615
+ def get_input_embeddings(self):
616
+ return self.word_embeddings
617
+
618
+ def _prepare_attn_mask(
619
+ self, attention_mask: torch.Tensor, input_shape: Tuple[int, int], past_key_values_length: int
620
+ ) -> torch.BoolTensor:
621
+ combined_attention_mask = None
622
+ device = attention_mask.device
623
+ _, src_length = input_shape
624
+
625
+ if src_length > 1:
626
+ combined_attention_mask = _make_causal_mask(
627
+ input_shape, device=device, past_key_values_length=past_key_values_length
628
+ )
629
+ expanded_attn_mask = _expand_mask(attention_mask, tgt_length=src_length)
630
+ combined_attention_mask = (
631
+ expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask | combined_attention_mask
632
+ )
633
+
634
+ return combined_attention_mask
635
+
636
+ def set_input_embeddings(self, new_embeddings: torch.Tensor):
637
+ self.word_embeddings = new_embeddings
638
+
639
+ def forward(
640
+ self,
641
+ input_ids: Optional[torch.LongTensor] = None,
642
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
643
+ attention_mask: Optional[torch.Tensor] = None,
644
+ inputs_embeds: Optional[torch.LongTensor] = None,
645
+ use_cache: Optional[bool] = None,
646
+ output_attentions: Optional[bool] = None,
647
+ output_hidden_states: Optional[bool] = None,
648
+ return_dict: Optional[bool] = None,
649
+ **deprecated_arguments,
650
+ ) -> Union[Tuple[torch.Tensor, ...], BaseModelOutputWithPastAndCrossAttentions]:
651
+
652
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
653
+ output_hidden_states = (
654
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
655
+ )
656
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
657
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
658
+
659
+ if input_ids is not None:
660
+ batch_size, seq_length = input_ids.shape
661
+ elif inputs_embeds is not None:
662
+ batch_size, seq_length, _ = inputs_embeds.shape
663
+
664
+ if past_key_values is None:
665
+ past_key_values = tuple([None] * len(self.h))
666
+
667
+ if inputs_embeds is None:
668
+ inputs_embeds = self.word_embeddings(input_ids)
669
+ hidden_states = inputs_embeds
670
+
671
+ if self.config.embed_layernorm:
672
+ hidden_states = self.word_embeddings_layernorm(inputs_embeds)
673
+
674
+ presents = () if use_cache else None
675
+ all_self_attentions = () if output_attentions else None
676
+ all_hidden_states = () if output_hidden_states else None
677
+
678
+ if self.gradient_checkpointing and self.training:
679
+ if use_cache:
680
+ use_cache = False
681
+
682
+ seq_length_with_past = seq_length
683
+ past_key_values_length = 0
684
+ if past_key_values[0] is not None:
685
+ past_key_values_length = past_key_values[0][0].shape[2]
686
+ seq_length_with_past = seq_length_with_past + past_key_values_length
687
+ if attention_mask is None:
688
+ attention_mask = torch.ones((batch_size, seq_length_with_past), device=hidden_states.device)
689
+ else:
690
+ attention_mask = attention_mask.to(hidden_states.device)
691
+ causal_mask = self._prepare_attn_mask(
692
+ attention_mask,
693
+ input_shape=(batch_size, seq_length),
694
+ past_key_values_length=past_key_values_length,
695
+ )
696
+
697
+ for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
698
+ if output_hidden_states:
699
+ all_hidden_states = all_hidden_states + (hidden_states,)
700
+
701
+ if self.gradient_checkpointing and self.training:
702
+
703
+ def create_custom_forward(module):
704
+ def custom_forward(*inputs):
705
+ # None for past_key_value
706
+ return module(*inputs, use_cache=use_cache, output_attentions=output_attentions)
707
+
708
+ return custom_forward
709
+
710
+ outputs = torch.utils.checkpoint.checkpoint(
711
+ create_custom_forward(block),
712
+ hidden_states,
713
+ causal_mask,
714
+ layer_past,
715
+ )
716
+ else:
717
+ outputs = block(
718
+ hidden_states,
719
+ layer_past=layer_past,
720
+ attention_mask=causal_mask,
721
+ use_cache=use_cache,
722
+ output_attentions=output_attentions,
723
+ )
724
+
725
+ hidden_states = outputs[0]
726
+ if use_cache is True:
727
+ presents = presents + (outputs[1],)
728
+
729
+ if output_attentions:
730
+ all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
731
+ hidden_states = self.ln_f(hidden_states)
732
+ if output_hidden_states:
733
+ all_hidden_states = all_hidden_states + (hidden_states,)
734
+ if not return_dict:
735
+ return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
736
+ return BaseModelOutputWithPastAndCrossAttentions(
737
+ last_hidden_state=hidden_states,
738
+ past_key_values=presents,
739
+ hidden_states=all_hidden_states,
740
+ attentions=all_self_attentions,
741
+ )
742
+
743
+
744
+ class TelechatForCausalLM(TelechatPreTrainedModel):
745
+ # _tied_weights_keys = ["lm_head.weight"]
746
+ _keys_to_ignore_on_load_missing = [r"lm_head.weight"]
747
+
748
+ def __init__(self, config: TelechatConfig):
749
+ super().__init__(config)
750
+ self.transformer = TelechatModel(config)
751
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
752
+ self.post_init()
753
+
754
+ def get_output_embeddings(self):
755
+ return self.lm_head
756
+
757
+ def set_output_embeddings(self, new_embeddings: torch.Tensor):
758
+ self.lm_head = new_embeddings
759
+
760
+ def prepare_inputs_for_generation(
761
+ self,
762
+ input_ids: torch.LongTensor,
763
+ past_key_values: Optional[torch.Tensor] = None,
764
+ attention_mask: Optional[torch.Tensor] = None,
765
+ inputs_embeds: Optional[torch.Tensor] = None,
766
+ **kwargs,
767
+ ) -> dict:
768
+ if past_key_values:
769
+ input_ids = input_ids[:, -1].unsqueeze(-1)
770
+ if inputs_embeds is not None and past_key_values is None:
771
+ model_inputs = {"inputs_embeds": inputs_embeds}
772
+ else:
773
+ model_inputs = {"input_ids": input_ids}
774
+
775
+ model_inputs.update(
776
+ {
777
+ "past_key_values": past_key_values,
778
+ "use_cache": kwargs.get("use_cache"),
779
+ "attention_mask": attention_mask,
780
+ }
781
+ )
782
+ return model_inputs
783
+
784
+ def forward(
785
+ self,
786
+ input_ids: Optional[torch.LongTensor] = None,
787
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
788
+ attention_mask: Optional[torch.Tensor] = None,
789
+ inputs_embeds: Optional[torch.Tensor] = None,
790
+ labels: Optional[torch.Tensor] = None,
791
+ use_cache: Optional[bool] = None,
792
+ output_attentions: Optional[bool] = None,
793
+ output_hidden_states: Optional[bool] = None,
794
+ return_dict: Optional[bool] = None,
795
+ **deprecated_arguments,
796
+ ) -> Union[Tuple[torch.Tensor], CausalLMOutputWithCrossAttentions]:
797
+
798
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
799
+
800
+ transformer_outputs = self.transformer(
801
+ input_ids,
802
+ past_key_values=past_key_values,
803
+ attention_mask=attention_mask,
804
+ inputs_embeds=inputs_embeds,
805
+ use_cache=use_cache,
806
+ output_attentions=output_attentions,
807
+ output_hidden_states=output_hidden_states,
808
+ return_dict=return_dict,
809
+ )
810
+ hidden_states = transformer_outputs[0]
811
+ lm_logits = self.lm_head(hidden_states)
812
+
813
+ loss = None
814
+ if labels is not None:
815
+ labels = labels.to(lm_logits.device)
816
+ shift_logits = lm_logits[..., :-1, :].contiguous()
817
+ shift_labels = labels[..., 1:].contiguous()
818
+ batch_size, seq_length, vocab_size = shift_logits.shape
819
+ loss_fct = CrossEntropyLoss()
820
+ loss = loss_fct(
821
+ shift_logits.view(batch_size * seq_length, vocab_size), shift_labels.view(batch_size * seq_length)
822
+ )
823
+
824
+ if not return_dict:
825
+ output = (lm_logits,) + transformer_outputs[1:]
826
+ return ((loss,) + output) if loss is not None else output
827
+
828
+ return CausalLMOutputWithCrossAttentions(
829
+ loss=loss,
830
+ logits=lm_logits,
831
+ past_key_values=transformer_outputs.past_key_values,
832
+ hidden_states=transformer_outputs.hidden_states,
833
+ attentions=transformer_outputs.attentions,
834
+ )
835
+
836
+ def chat(self, tokenizer, question: str = '', history: Union[List[Dict], History] = None, stream: bool = False,
837
+ generation_config: Optional[GenerationConfig] = None, **kwargs):
838
+ """
839
+ Args:
840
+ tokenizer: the tokenizer of telechat
841
+ question: question which the model reply in this turn
842
+ history: history which will format the input for telechat
843
+ stream: if return the full text at last or yield the text in token
844
+ generation_config: configuration for generation
845
+ **kwargs: args which will update the generation config or pass to model forward
846
+ """
847
+ generation_config = generation_config or self.generation_config
848
+ if not generation_config:
849
+ logger.error("generation_config is None")
850
+ raise ValueError("generation_config must not be None")
851
+ if not question:
852
+ logger.error("question is empty")
853
+ raise ValueError("question must not be empty")
854
+ if history is None:
855
+ history = []
856
+
857
+ # we update and check generate_config here for building inputs.
858
+
859
+ generation_config = copy.deepcopy(generation_config)
860
+ user_id = generation_config.user_token_id
861
+ bot_id = generation_config.bot_token_id
862
+ model_kwargs = generation_config.update(**kwargs)
863
+ generation_config.validate()
864
+
865
+ # transfer to History
866
+ if not isinstance(history, History):
867
+ history = History(tokenizer, history)
868
+
869
+ inputs = self.build_inputs_for_chat(tokenizer, question, history, generation_config, user_id, bot_id)
870
+ history.append({"role": "user", "content": question})
871
+ if stream:
872
+ streamer = TelechatIterTextStreamer(tokenizer, history,skip_prompt=True)
873
+ Thread(target=self.generate, kwargs=dict(
874
+ inputs=inputs.to(self.device), streamer=streamer,
875
+ generation_config=generation_config, **model_kwargs
876
+ )).start()
877
+ return streamer
878
+ else:
879
+ outputs = self.generate(inputs.to(self.device), generation_config=generation_config, **model_kwargs)
880
+ response = tokenizer.decode(outputs[0][len(inputs[0]):-1])
881
+ history.append({"role": "bot", "content": response})
882
+ return response, history
883
+
884
+ def build_inputs_for_chat(self, tokenizer, question, history, generation_config, usr_id, bot_id):
885
+ """
886
+ check history and build inputs here
887
+ """
888
+ # first tokenize question
889
+ q_token = tokenizer(question)
890
+ qa_history = copy.deepcopy(history)
891
+
892
+ # get the max length we should build our inputs in
893
+ model_max_length = self.config.seq_length
894
+ build_max_length = max(0, model_max_length - generation_config.max_new_tokens) \
895
+ if generation_config.max_new_tokens else max(0, generation_config.max_length)
896
+ if build_max_length < 3:
897
+ logger.warning("the model can not meet the requirements of input length,Please check config")
898
+ raise ValueError("")
899
+
900
+ # trunc left
901
+ input_tokens = [usr_id] + q_token["input_ids"][-build_max_length + 1:] + [bot_id]
902
+ length = len(input_tokens)
903
+
904
+ while len(qa_history) != 0:
905
+ message = qa_history.pop()
906
+ if message["role"] == "user":
907
+ tokens = [usr_id] + message["input_ids"]
908
+ elif message["role"] == "bot":
909
+ tokens = [bot_id] + message["input_ids"] + [generation_config.eos_token_id]
910
+ else:
911
+ tokens = []
912
+ if len(tokens) + length >= build_max_length:
913
+ break
914
+ else:
915
+ input_tokens = tokens + input_tokens
916
+
917
+ return torch.tensor([input_tokens], dtype=torch.int64)
quantize_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bits": 8,
3
+ "group_size": 128,
4
+ "damp_percent": 0.01,
5
+ "desc_act": false,
6
+ "sym": true,
7
+ "true_sequential": true,
8
+ "model_name_or_path": null,
9
+ "model_file_base_name": null
10
+ }
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<_start>", "eos_token": "<_end>", "unk_token": "<unk>", "pad_token": "<pad>"}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "<unk>", "eos_token": "<_end>", "bos_token": "<_start>", "pad_token": "<pad>", "name_or_path": "bigscience/tokenizer", "special_tokens_map_file": null, "tokenizer_class": "BloomTokenizerFast", "padding_side":"left"}