Hugging Face's logo

moussaKam
/
fr-llama-1B-instruct

Safetensors
llama
Model card Files Community
moussaKam commited on
Commit
542b851
·
verified ·
1 Parent(s): f5a998d

upload

Browse files
Files changed (11) hide show
  1. .gitattributes +1 -0
  2. config.json +36 -0
  3. generation_config.json +9 -0
  4. latest +1 -0
  5. model.safetensors +3 -0
  6. special_tokens_map.json +23 -0
  7. tokenizer.json +3 -0
  8. tokenizer_config.json +2065 -0
  9. trainer_state.json +3743 -0
  10. training_args.bin +3 -0
  11. zero_to_fp32.py +674 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/lustre/fsn1/projects/rech/gkb/uua32zb/grand_challenge/checkpoints/meta-llama__Meta-Llama-3.2-1B-annealing2-0.0002LR-8192CL-2GAS-2BS-1EPOCHS-0.9BETA1-0.95BETA2/",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": 128001,
10
+ "head_dim": 64,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 2048,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 8192,
15
+ "max_position_embeddings": 131072,
16
+ "mlp_bias": false,
17
+ "model_type": "llama",
18
+ "num_attention_heads": 32,
19
+ "num_hidden_layers": 16,
20
+ "num_key_value_heads": 8,
21
+ "pretraining_tp": 1,
22
+ "rms_norm_eps": 1e-05,
23
+ "rope_scaling": {
24
+ "factor": 32.0,
25
+ "high_freq_factor": 4.0,
26
+ "low_freq_factor": 1.0,
27
+ "original_max_position_embeddings": 8192,
28
+ "rope_type": "llama3"
29
+ },
30
+ "rope_theta": 500000.0,
31
+ "tie_word_embeddings": true,
32
+ "torch_dtype": "bfloat16",
33
+ "transformers_version": "4.46.1",
34
+ "use_cache": false,
35
+ "vocab_size": 128256
36
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 128000,
4
+ "do_sample": true,
5
+ "eos_token_id": 128001,
6
+ "temperature": 0.6,
7
+ "top_p": 0.9,
8
+ "transformers_version": "4.46.1"
9
+ }
latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step26534
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c7a3966f4afbe1dbf9f589ed2a5a6fa2db6486012634705aa3ec8a4ed42dcd7
3
+ size 2996982344
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<|begin_of_text|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|eot_id|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<|eot_id|>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
3
+ size 17209920
tokenizer_config.json ADDED
@@ -0,0 +1,2065 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "128000": {
4
+ "content": "<|begin_of_text|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "128001": {
12
+ "content": "<|end_of_text|>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "128002": {
20
+ "content": "<|reserved_special_token_0|>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "128003": {
28
+ "content": "<|reserved_special_token_1|>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "128004": {
36
+ "content": "<|finetune_right_pad_id|>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "128005": {
44
+ "content": "<|reserved_special_token_2|>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "128006": {
52
+ "content": "<|start_header_id|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "128007": {
60
+ "content": "<|end_header_id|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "128008": {
68
+ "content": "<|eom_id|>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "128009": {
76
+ "content": "<|eot_id|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "128010": {
84
+ "content": "<|python_tag|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ },
91
+ "128011": {
92
+ "content": "<|reserved_special_token_3|>",
93
+ "lstrip": false,
94
+ "normalized": false,
95
+ "rstrip": false,
96
+ "single_word": false,
97
+ "special": true
98
+ },
99
+ "128012": {
100
+ "content": "<|reserved_special_token_4|>",
101
+ "lstrip": false,
102
+ "normalized": false,
103
+ "rstrip": false,
104
+ "single_word": false,
105
+ "special": true
106
+ },
107
+ "128013": {
108
+ "content": "<|reserved_special_token_5|>",
109
+ "lstrip": false,
110
+ "normalized": false,
111
+ "rstrip": false,
112
+ "single_word": false,
113
+ "special": true
114
+ },
115
+ "128014": {
116
+ "content": "<|reserved_special_token_6|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false,
121
+ "special": true
122
+ },
123
+ "128015": {
124
+ "content": "<|reserved_special_token_7|>",
125
+ "lstrip": false,
126
+ "normalized": false,
127
+ "rstrip": false,
128
+ "single_word": false,
129
+ "special": true
130
+ },
131
+ "128016": {
132
+ "content": "<|reserved_special_token_8|>",
133
+ "lstrip": false,
134
+ "normalized": false,
135
+ "rstrip": false,
136
+ "single_word": false,
137
+ "special": true
138
+ },
139
+ "128017": {
140
+ "content": "<|reserved_special_token_9|>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false,
145
+ "special": true
146
+ },
147
+ "128018": {
148
+ "content": "<|reserved_special_token_10|>",
149
+ "lstrip": false,
150
+ "normalized": false,
151
+ "rstrip": false,
152
+ "single_word": false,
153
+ "special": true
154
+ },
155
+ "128019": {
156
+ "content": "<|reserved_special_token_11|>",
157
+ "lstrip": false,
158
+ "normalized": false,
159
+ "rstrip": false,
160
+ "single_word": false,
161
+ "special": true
162
+ },
163
+ "128020": {
164
+ "content": "<|reserved_special_token_12|>",
165
+ "lstrip": false,
166
+ "normalized": false,
167
+ "rstrip": false,
168
+ "single_word": false,
169
+ "special": true
170
+ },
171
+ "128021": {
172
+ "content": "<|reserved_special_token_13|>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false,
177
+ "special": true
178
+ },
179
+ "128022": {
180
+ "content": "<|reserved_special_token_14|>",
181
+ "lstrip": false,
182
+ "normalized": false,
183
+ "rstrip": false,
184
+ "single_word": false,
185
+ "special": true
186
+ },
187
+ "128023": {
188
+ "content": "<|reserved_special_token_15|>",
189
+ "lstrip": false,
190
+ "normalized": false,
191
+ "rstrip": false,
192
+ "single_word": false,
193
+ "special": true
194
+ },
195
+ "128024": {
196
+ "content": "<|reserved_special_token_16|>",
197
+ "lstrip": false,
198
+ "normalized": false,
199
+ "rstrip": false,
200
+ "single_word": false,
201
+ "special": true
202
+ },
203
+ "128025": {
204
+ "content": "<|reserved_special_token_17|>",
205
+ "lstrip": false,
206
+ "normalized": false,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": true
210
+ },
211
+ "128026": {
212
+ "content": "<|reserved_special_token_18|>",
213
+ "lstrip": false,
214
+ "normalized": false,
215
+ "rstrip": false,
216
+ "single_word": false,
217
+ "special": true
218
+ },
219
+ "128027": {
220
+ "content": "<|reserved_special_token_19|>",
221
+ "lstrip": false,
222
+ "normalized": false,
223
+ "rstrip": false,
224
+ "single_word": false,
225
+ "special": true
226
+ },
227
+ "128028": {
228
+ "content": "<|reserved_special_token_20|>",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false,
233
+ "special": true
234
+ },
235
+ "128029": {
236
+ "content": "<|reserved_special_token_21|>",
237
+ "lstrip": false,
238
+ "normalized": false,
239
+ "rstrip": false,
240
+ "single_word": false,
241
+ "special": true
242
+ },
243
+ "128030": {
244
+ "content": "<|reserved_special_token_22|>",
245
+ "lstrip": false,
246
+ "normalized": false,
247
+ "rstrip": false,
248
+ "single_word": false,
249
+ "special": true
250
+ },
251
+ "128031": {
252
+ "content": "<|reserved_special_token_23|>",
253
+ "lstrip": false,
254
+ "normalized": false,
255
+ "rstrip": false,
256
+ "single_word": false,
257
+ "special": true
258
+ },
259
+ "128032": {
260
+ "content": "<|reserved_special_token_24|>",
261
+ "lstrip": false,
262
+ "normalized": false,
263
+ "rstrip": false,
264
+ "single_word": false,
265
+ "special": true
266
+ },
267
+ "128033": {
268
+ "content": "<|reserved_special_token_25|>",
269
+ "lstrip": false,
270
+ "normalized": false,
271
+ "rstrip": false,
272
+ "single_word": false,
273
+ "special": true
274
+ },
275
+ "128034": {
276
+ "content": "<|reserved_special_token_26|>",
277
+ "lstrip": false,
278
+ "normalized": false,
279
+ "rstrip": false,
280
+ "single_word": false,
281
+ "special": true
282
+ },
283
+ "128035": {
284
+ "content": "<|reserved_special_token_27|>",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false,
289
+ "special": true
290
+ },
291
+ "128036": {
292
+ "content": "<|reserved_special_token_28|>",
293
+ "lstrip": false,
294
+ "normalized": false,
295
+ "rstrip": false,
296
+ "single_word": false,
297
+ "special": true
298
+ },
299
+ "128037": {
300
+ "content": "<|reserved_special_token_29|>",
301
+ "lstrip": false,
302
+ "normalized": false,
303
+ "rstrip": false,
304
+ "single_word": false,
305
+ "special": true
306
+ },
307
+ "128038": {
308
+ "content": "<|reserved_special_token_30|>",
309
+ "lstrip": false,
310
+ "normalized": false,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": true
314
+ },
315
+ "128039": {
316
+ "content": "<|reserved_special_token_31|>",
317
+ "lstrip": false,
318
+ "normalized": false,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": true
322
+ },
323
+ "128040": {
324
+ "content": "<|reserved_special_token_32|>",
325
+ "lstrip": false,
326
+ "normalized": false,
327
+ "rstrip": false,
328
+ "single_word": false,
329
+ "special": true
330
+ },
331
+ "128041": {
332
+ "content": "<|reserved_special_token_33|>",
333
+ "lstrip": false,
334
+ "normalized": false,
335
+ "rstrip": false,
336
+ "single_word": false,
337
+ "special": true
338
+ },
339
+ "128042": {
340
+ "content": "<|reserved_special_token_34|>",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false,
345
+ "special": true
346
+ },
347
+ "128043": {
348
+ "content": "<|reserved_special_token_35|>",
349
+ "lstrip": false,
350
+ "normalized": false,
351
+ "rstrip": false,
352
+ "single_word": false,
353
+ "special": true
354
+ },
355
+ "128044": {
356
+ "content": "<|reserved_special_token_36|>",
357
+ "lstrip": false,
358
+ "normalized": false,
359
+ "rstrip": false,
360
+ "single_word": false,
361
+ "special": true
362
+ },
363
+ "128045": {
364
+ "content": "<|reserved_special_token_37|>",
365
+ "lstrip": false,
366
+ "normalized": false,
367
+ "rstrip": false,
368
+ "single_word": false,
369
+ "special": true
370
+ },
371
+ "128046": {
372
+ "content": "<|reserved_special_token_38|>",
373
+ "lstrip": false,
374
+ "normalized": false,
375
+ "rstrip": false,
376
+ "single_word": false,
377
+ "special": true
378
+ },
379
+ "128047": {
380
+ "content": "<|reserved_special_token_39|>",
381
+ "lstrip": false,
382
+ "normalized": false,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": true
386
+ },
387
+ "128048": {
388
+ "content": "<|reserved_special_token_40|>",
389
+ "lstrip": false,
390
+ "normalized": false,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": true
394
+ },
395
+ "128049": {
396
+ "content": "<|reserved_special_token_41|>",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": true
402
+ },
403
+ "128050": {
404
+ "content": "<|reserved_special_token_42|>",
405
+ "lstrip": false,
406
+ "normalized": false,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": true
410
+ },
411
+ "128051": {
412
+ "content": "<|reserved_special_token_43|>",
413
+ "lstrip": false,
414
+ "normalized": false,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": true
418
+ },
419
+ "128052": {
420
+ "content": "<|reserved_special_token_44|>",
421
+ "lstrip": false,
422
+ "normalized": false,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": true
426
+ },
427
+ "128053": {
428
+ "content": "<|reserved_special_token_45|>",
429
+ "lstrip": false,
430
+ "normalized": false,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": true
434
+ },
435
+ "128054": {
436
+ "content": "<|reserved_special_token_46|>",
437
+ "lstrip": false,
438
+ "normalized": false,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": true
442
+ },
443
+ "128055": {
444
+ "content": "<|reserved_special_token_47|>",
445
+ "lstrip": false,
446
+ "normalized": false,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": true
450
+ },
451
+ "128056": {
452
+ "content": "<|reserved_special_token_48|>",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": true
458
+ },
459
+ "128057": {
460
+ "content": "<|reserved_special_token_49|>",
461
+ "lstrip": false,
462
+ "normalized": false,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": true
466
+ },
467
+ "128058": {
468
+ "content": "<|reserved_special_token_50|>",
469
+ "lstrip": false,
470
+ "normalized": false,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": true
474
+ },
475
+ "128059": {
476
+ "content": "<|reserved_special_token_51|>",
477
+ "lstrip": false,
478
+ "normalized": false,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": true
482
+ },
483
+ "128060": {
484
+ "content": "<|reserved_special_token_52|>",
485
+ "lstrip": false,
486
+ "normalized": false,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": true
490
+ },
491
+ "128061": {
492
+ "content": "<|reserved_special_token_53|>",
493
+ "lstrip": false,
494
+ "normalized": false,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": true
498
+ },
499
+ "128062": {
500
+ "content": "<|reserved_special_token_54|>",
501
+ "lstrip": false,
502
+ "normalized": false,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": true
506
+ },
507
+ "128063": {
508
+ "content": "<|reserved_special_token_55|>",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": true
514
+ },
515
+ "128064": {
516
+ "content": "<|reserved_special_token_56|>",
517
+ "lstrip": false,
518
+ "normalized": false,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": true
522
+ },
523
+ "128065": {
524
+ "content": "<|reserved_special_token_57|>",
525
+ "lstrip": false,
526
+ "normalized": false,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": true
530
+ },
531
+ "128066": {
532
+ "content": "<|reserved_special_token_58|>",
533
+ "lstrip": false,
534
+ "normalized": false,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": true
538
+ },
539
+ "128067": {
540
+ "content": "<|reserved_special_token_59|>",
541
+ "lstrip": false,
542
+ "normalized": false,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": true
546
+ },
547
+ "128068": {
548
+ "content": "<|reserved_special_token_60|>",
549
+ "lstrip": false,
550
+ "normalized": false,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": true
554
+ },
555
+ "128069": {
556
+ "content": "<|reserved_special_token_61|>",
557
+ "lstrip": false,
558
+ "normalized": false,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": true
562
+ },
563
+ "128070": {
564
+ "content": "<|reserved_special_token_62|>",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": true
570
+ },
571
+ "128071": {
572
+ "content": "<|reserved_special_token_63|>",
573
+ "lstrip": false,
574
+ "normalized": false,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": true
578
+ },
579
+ "128072": {
580
+ "content": "<|reserved_special_token_64|>",
581
+ "lstrip": false,
582
+ "normalized": false,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": true
586
+ },
587
+ "128073": {
588
+ "content": "<|reserved_special_token_65|>",
589
+ "lstrip": false,
590
+ "normalized": false,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": true
594
+ },
595
+ "128074": {
596
+ "content": "<|reserved_special_token_66|>",
597
+ "lstrip": false,
598
+ "normalized": false,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": true
602
+ },
603
+ "128075": {
604
+ "content": "<|reserved_special_token_67|>",
605
+ "lstrip": false,
606
+ "normalized": false,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": true
610
+ },
611
+ "128076": {
612
+ "content": "<|reserved_special_token_68|>",
613
+ "lstrip": false,
614
+ "normalized": false,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": true
618
+ },
619
+ "128077": {
620
+ "content": "<|reserved_special_token_69|>",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": true
626
+ },
627
+ "128078": {
628
+ "content": "<|reserved_special_token_70|>",
629
+ "lstrip": false,
630
+ "normalized": false,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": true
634
+ },
635
+ "128079": {
636
+ "content": "<|reserved_special_token_71|>",
637
+ "lstrip": false,
638
+ "normalized": false,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": true
642
+ },
643
+ "128080": {
644
+ "content": "<|reserved_special_token_72|>",
645
+ "lstrip": false,
646
+ "normalized": false,
647
+ "rstrip": false,
648
+ "single_word": false,
649
+ "special": true
650
+ },
651
+ "128081": {
652
+ "content": "<|reserved_special_token_73|>",
653
+ "lstrip": false,
654
+ "normalized": false,
655
+ "rstrip": false,
656
+ "single_word": false,
657
+ "special": true
658
+ },
659
+ "128082": {
660
+ "content": "<|reserved_special_token_74|>",
661
+ "lstrip": false,
662
+ "normalized": false,
663
+ "rstrip": false,
664
+ "single_word": false,
665
+ "special": true
666
+ },
667
+ "128083": {
668
+ "content": "<|reserved_special_token_75|>",
669
+ "lstrip": false,
670
+ "normalized": false,
671
+ "rstrip": false,
672
+ "single_word": false,
673
+ "special": true
674
+ },
675
+ "128084": {
676
+ "content": "<|reserved_special_token_76|>",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false,
681
+ "special": true
682
+ },
683
+ "128085": {
684
+ "content": "<|reserved_special_token_77|>",
685
+ "lstrip": false,
686
+ "normalized": false,
687
+ "rstrip": false,
688
+ "single_word": false,
689
+ "special": true
690
+ },
691
+ "128086": {
692
+ "content": "<|reserved_special_token_78|>",
693
+ "lstrip": false,
694
+ "normalized": false,
695
+ "rstrip": false,
696
+ "single_word": false,
697
+ "special": true
698
+ },
699
+ "128087": {
700
+ "content": "<|reserved_special_token_79|>",
701
+ "lstrip": false,
702
+ "normalized": false,
703
+ "rstrip": false,
704
+ "single_word": false,
705
+ "special": true
706
+ },
707
+ "128088": {
708
+ "content": "<|reserved_special_token_80|>",
709
+ "lstrip": false,
710
+ "normalized": false,
711
+ "rstrip": false,
712
+ "single_word": false,
713
+ "special": true
714
+ },
715
+ "128089": {
716
+ "content": "<|reserved_special_token_81|>",
717
+ "lstrip": false,
718
+ "normalized": false,
719
+ "rstrip": false,
720
+ "single_word": false,
721
+ "special": true
722
+ },
723
+ "128090": {
724
+ "content": "<|reserved_special_token_82|>",
725
+ "lstrip": false,
726
+ "normalized": false,
727
+ "rstrip": false,
728
+ "single_word": false,
729
+ "special": true
730
+ },
731
+ "128091": {
732
+ "content": "<|reserved_special_token_83|>",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false,
737
+ "special": true
738
+ },
739
+ "128092": {
740
+ "content": "<|reserved_special_token_84|>",
741
+ "lstrip": false,
742
+ "normalized": false,
743
+ "rstrip": false,
744
+ "single_word": false,
745
+ "special": true
746
+ },
747
+ "128093": {
748
+ "content": "<|reserved_special_token_85|>",
749
+ "lstrip": false,
750
+ "normalized": false,
751
+ "rstrip": false,
752
+ "single_word": false,
753
+ "special": true
754
+ },
755
+ "128094": {
756
+ "content": "<|reserved_special_token_86|>",
757
+ "lstrip": false,
758
+ "normalized": false,
759
+ "rstrip": false,
760
+ "single_word": false,
761
+ "special": true
762
+ },
763
+ "128095": {
764
+ "content": "<|reserved_special_token_87|>",
765
+ "lstrip": false,
766
+ "normalized": false,
767
+ "rstrip": false,
768
+ "single_word": false,
769
+ "special": true
770
+ },
771
+ "128096": {
772
+ "content": "<|reserved_special_token_88|>",
773
+ "lstrip": false,
774
+ "normalized": false,
775
+ "rstrip": false,
776
+ "single_word": false,
777
+ "special": true
778
+ },
779
+ "128097": {
780
+ "content": "<|reserved_special_token_89|>",
781
+ "lstrip": false,
782
+ "normalized": false,
783
+ "rstrip": false,
784
+ "single_word": false,
785
+ "special": true
786
+ },
787
+ "128098": {
788
+ "content": "<|reserved_special_token_90|>",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false,
793
+ "special": true
794
+ },
795
+ "128099": {
796
+ "content": "<|reserved_special_token_91|>",
797
+ "lstrip": false,
798
+ "normalized": false,
799
+ "rstrip": false,
800
+ "single_word": false,
801
+ "special": true
802
+ },
803
+ "128100": {
804
+ "content": "<|reserved_special_token_92|>",
805
+ "lstrip": false,
806
+ "normalized": false,
807
+ "rstrip": false,
808
+ "single_word": false,
809
+ "special": true
810
+ },
811
+ "128101": {
812
+ "content": "<|reserved_special_token_93|>",
813
+ "lstrip": false,
814
+ "normalized": false,
815
+ "rstrip": false,
816
+ "single_word": false,
817
+ "special": true
818
+ },
819
+ "128102": {
820
+ "content": "<|reserved_special_token_94|>",
821
+ "lstrip": false,
822
+ "normalized": false,
823
+ "rstrip": false,
824
+ "single_word": false,
825
+ "special": true
826
+ },
827
+ "128103": {
828
+ "content": "<|reserved_special_token_95|>",
829
+ "lstrip": false,
830
+ "normalized": false,
831
+ "rstrip": false,
832
+ "single_word": false,
833
+ "special": true
834
+ },
835
+ "128104": {
836
+ "content": "<|reserved_special_token_96|>",
837
+ "lstrip": false,
838
+ "normalized": false,
839
+ "rstrip": false,
840
+ "single_word": false,
841
+ "special": true
842
+ },
843
+ "128105": {
844
+ "content": "<|reserved_special_token_97|>",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false,
849
+ "special": true
850
+ },
851
+ "128106": {
852
+ "content": "<|reserved_special_token_98|>",
853
+ "lstrip": false,
854
+ "normalized": false,
855
+ "rstrip": false,
856
+ "single_word": false,
857
+ "special": true
858
+ },
859
+ "128107": {
860
+ "content": "<|reserved_special_token_99|>",
861
+ "lstrip": false,
862
+ "normalized": false,
863
+ "rstrip": false,
864
+ "single_word": false,
865
+ "special": true
866
+ },
867
+ "128108": {
868
+ "content": "<|reserved_special_token_100|>",
869
+ "lstrip": false,
870
+ "normalized": false,
871
+ "rstrip": false,
872
+ "single_word": false,
873
+ "special": true
874
+ },
875
+ "128109": {
876
+ "content": "<|reserved_special_token_101|>",
877
+ "lstrip": false,
878
+ "normalized": false,
879
+ "rstrip": false,
880
+ "single_word": false,
881
+ "special": true
882
+ },
883
+ "128110": {
884
+ "content": "<|reserved_special_token_102|>",
885
+ "lstrip": false,
886
+ "normalized": false,
887
+ "rstrip": false,
888
+ "single_word": false,
889
+ "special": true
890
+ },
891
+ "128111": {
892
+ "content": "<|reserved_special_token_103|>",
893
+ "lstrip": false,
894
+ "normalized": false,
895
+ "rstrip": false,
896
+ "single_word": false,
897
+ "special": true
898
+ },
899
+ "128112": {
900
+ "content": "<|reserved_special_token_104|>",
901
+ "lstrip": false,
902
+ "normalized": false,
903
+ "rstrip": false,
904
+ "single_word": false,
905
+ "special": true
906
+ },
907
+ "128113": {
908
+ "content": "<|reserved_special_token_105|>",
909
+ "lstrip": false,
910
+ "normalized": false,
911
+ "rstrip": false,
912
+ "single_word": false,
913
+ "special": true
914
+ },
915
+ "128114": {
916
+ "content": "<|reserved_special_token_106|>",
917
+ "lstrip": false,
918
+ "normalized": false,
919
+ "rstrip": false,
920
+ "single_word": false,
921
+ "special": true
922
+ },
923
+ "128115": {
924
+ "content": "<|reserved_special_token_107|>",
925
+ "lstrip": false,
926
+ "normalized": false,
927
+ "rstrip": false,
928
+ "single_word": false,
929
+ "special": true
930
+ },
931
+ "128116": {
932
+ "content": "<|reserved_special_token_108|>",
933
+ "lstrip": false,
934
+ "normalized": false,
935
+ "rstrip": false,
936
+ "single_word": false,
937
+ "special": true
938
+ },
939
+ "128117": {
940
+ "content": "<|reserved_special_token_109|>",
941
+ "lstrip": false,
942
+ "normalized": false,
943
+ "rstrip": false,
944
+ "single_word": false,
945
+ "special": true
946
+ },
947
+ "128118": {
948
+ "content": "<|reserved_special_token_110|>",
949
+ "lstrip": false,
950
+ "normalized": false,
951
+ "rstrip": false,
952
+ "single_word": false,
953
+ "special": true
954
+ },
955
+ "128119": {
956
+ "content": "<|reserved_special_token_111|>",
957
+ "lstrip": false,
958
+ "normalized": false,
959
+ "rstrip": false,
960
+ "single_word": false,
961
+ "special": true
962
+ },
963
+ "128120": {
964
+ "content": "<|reserved_special_token_112|>",
965
+ "lstrip": false,
966
+ "normalized": false,
967
+ "rstrip": false,
968
+ "single_word": false,
969
+ "special": true
970
+ },
971
+ "128121": {
972
+ "content": "<|reserved_special_token_113|>",
973
+ "lstrip": false,
974
+ "normalized": false,
975
+ "rstrip": false,
976
+ "single_word": false,
977
+ "special": true
978
+ },
979
+ "128122": {
980
+ "content": "<|reserved_special_token_114|>",
981
+ "lstrip": false,
982
+ "normalized": false,
983
+ "rstrip": false,
984
+ "single_word": false,
985
+ "special": true
986
+ },
987
+ "128123": {
988
+ "content": "<|reserved_special_token_115|>",
989
+ "lstrip": false,
990
+ "normalized": false,
991
+ "rstrip": false,
992
+ "single_word": false,
993
+ "special": true
994
+ },
995
+ "128124": {
996
+ "content": "<|reserved_special_token_116|>",
997
+ "lstrip": false,
998
+ "normalized": false,
999
+ "rstrip": false,
1000
+ "single_word": false,
1001
+ "special": true
1002
+ },
1003
+ "128125": {
1004
+ "content": "<|reserved_special_token_117|>",
1005
+ "lstrip": false,
1006
+ "normalized": false,
1007
+ "rstrip": false,
1008
+ "single_word": false,
1009
+ "special": true
1010
+ },
1011
+ "128126": {
1012
+ "content": "<|reserved_special_token_118|>",
1013
+ "lstrip": false,
1014
+ "normalized": false,
1015
+ "rstrip": false,
1016
+ "single_word": false,
1017
+ "special": true
1018
+ },
1019
+ "128127": {
1020
+ "content": "<|reserved_special_token_119|>",
1021
+ "lstrip": false,
1022
+ "normalized": false,
1023
+ "rstrip": false,
1024
+ "single_word": false,
1025
+ "special": true
1026
+ },
1027
+ "128128": {
1028
+ "content": "<|reserved_special_token_120|>",
1029
+ "lstrip": false,
1030
+ "normalized": false,
1031
+ "rstrip": false,
1032
+ "single_word": false,
1033
+ "special": true
1034
+ },
1035
+ "128129": {
1036
+ "content": "<|reserved_special_token_121|>",
1037
+ "lstrip": false,
1038
+ "normalized": false,
1039
+ "rstrip": false,
1040
+ "single_word": false,
1041
+ "special": true
1042
+ },
1043
+ "128130": {
1044
+ "content": "<|reserved_special_token_122|>",
1045
+ "lstrip": false,
1046
+ "normalized": false,
1047
+ "rstrip": false,
1048
+ "single_word": false,
1049
+ "special": true
1050
+ },
1051
+ "128131": {
1052
+ "content": "<|reserved_special_token_123|>",
1053
+ "lstrip": false,
1054
+ "normalized": false,
1055
+ "rstrip": false,
1056
+ "single_word": false,
1057
+ "special": true
1058
+ },
1059
+ "128132": {
1060
+ "content": "<|reserved_special_token_124|>",
1061
+ "lstrip": false,
1062
+ "normalized": false,
1063
+ "rstrip": false,
1064
+ "single_word": false,
1065
+ "special": true
1066
+ },
1067
+ "128133": {
1068
+ "content": "<|reserved_special_token_125|>",
1069
+ "lstrip": false,
1070
+ "normalized": false,
1071
+ "rstrip": false,
1072
+ "single_word": false,
1073
+ "special": true
1074
+ },
1075
+ "128134": {
1076
+ "content": "<|reserved_special_token_126|>",
1077
+ "lstrip": false,
1078
+ "normalized": false,
1079
+ "rstrip": false,
1080
+ "single_word": false,
1081
+ "special": true
1082
+ },
1083
+ "128135": {
1084
+ "content": "<|reserved_special_token_127|>",
1085
+ "lstrip": false,
1086
+ "normalized": false,
1087
+ "rstrip": false,
1088
+ "single_word": false,
1089
+ "special": true
1090
+ },
1091
+ "128136": {
1092
+ "content": "<|reserved_special_token_128|>",
1093
+ "lstrip": false,
1094
+ "normalized": false,
1095
+ "rstrip": false,
1096
+ "single_word": false,
1097
+ "special": true
1098
+ },
1099
+ "128137": {
1100
+ "content": "<|reserved_special_token_129|>",
1101
+ "lstrip": false,
1102
+ "normalized": false,
1103
+ "rstrip": false,
1104
+ "single_word": false,
1105
+ "special": true
1106
+ },
1107
+ "128138": {
1108
+ "content": "<|reserved_special_token_130|>",
1109
+ "lstrip": false,
1110
+ "normalized": false,
1111
+ "rstrip": false,
1112
+ "single_word": false,
1113
+ "special": true
1114
+ },
1115
+ "128139": {
1116
+ "content": "<|reserved_special_token_131|>",
1117
+ "lstrip": false,
1118
+ "normalized": false,
1119
+ "rstrip": false,
1120
+ "single_word": false,
1121
+ "special": true
1122
+ },
1123
+ "128140": {
1124
+ "content": "<|reserved_special_token_132|>",
1125
+ "lstrip": false,
1126
+ "normalized": false,
1127
+ "rstrip": false,
1128
+ "single_word": false,
1129
+ "special": true
1130
+ },
1131
+ "128141": {
1132
+ "content": "<|reserved_special_token_133|>",
1133
+ "lstrip": false,
1134
+ "normalized": false,
1135
+ "rstrip": false,
1136
+ "single_word": false,
1137
+ "special": true
1138
+ },
1139
+ "128142": {
1140
+ "content": "<|reserved_special_token_134|>",
1141
+ "lstrip": false,
1142
+ "normalized": false,
1143
+ "rstrip": false,
1144
+ "single_word": false,
1145
+ "special": true
1146
+ },
1147
+ "128143": {
1148
+ "content": "<|reserved_special_token_135|>",
1149
+ "lstrip": false,
1150
+ "normalized": false,
1151
+ "rstrip": false,
1152
+ "single_word": false,
1153
+ "special": true
1154
+ },
1155
+ "128144": {
1156
+ "content": "<|reserved_special_token_136|>",
1157
+ "lstrip": false,
1158
+ "normalized": false,
1159
+ "rstrip": false,
1160
+ "single_word": false,
1161
+ "special": true
1162
+ },
1163
+ "128145": {
1164
+ "content": "<|reserved_special_token_137|>",
1165
+ "lstrip": false,
1166
+ "normalized": false,
1167
+ "rstrip": false,
1168
+ "single_word": false,
1169
+ "special": true
1170
+ },
1171
+ "128146": {
1172
+ "content": "<|reserved_special_token_138|>",
1173
+ "lstrip": false,
1174
+ "normalized": false,
1175
+ "rstrip": false,
1176
+ "single_word": false,
1177
+ "special": true
1178
+ },
1179
+ "128147": {
1180
+ "content": "<|reserved_special_token_139|>",
1181
+ "lstrip": false,
1182
+ "normalized": false,
1183
+ "rstrip": false,
1184
+ "single_word": false,
1185
+ "special": true
1186
+ },
1187
+ "128148": {
1188
+ "content": "<|reserved_special_token_140|>",
1189
+ "lstrip": false,
1190
+ "normalized": false,
1191
+ "rstrip": false,
1192
+ "single_word": false,
1193
+ "special": true
1194
+ },
1195
+ "128149": {
1196
+ "content": "<|reserved_special_token_141|>",
1197
+ "lstrip": false,
1198
+ "normalized": false,
1199
+ "rstrip": false,
1200
+ "single_word": false,
1201
+ "special": true
1202
+ },
1203
+ "128150": {
1204
+ "content": "<|reserved_special_token_142|>",
1205
+ "lstrip": false,
1206
+ "normalized": false,
1207
+ "rstrip": false,
1208
+ "single_word": false,
1209
+ "special": true
1210
+ },
1211
+ "128151": {
1212
+ "content": "<|reserved_special_token_143|>",
1213
+ "lstrip": false,
1214
+ "normalized": false,
1215
+ "rstrip": false,
1216
+ "single_word": false,
1217
+ "special": true
1218
+ },
1219
+ "128152": {
1220
+ "content": "<|reserved_special_token_144|>",
1221
+ "lstrip": false,
1222
+ "normalized": false,
1223
+ "rstrip": false,
1224
+ "single_word": false,
1225
+ "special": true
1226
+ },
1227
+ "128153": {
1228
+ "content": "<|reserved_special_token_145|>",
1229
+ "lstrip": false,
1230
+ "normalized": false,
1231
+ "rstrip": false,
1232
+ "single_word": false,
1233
+ "special": true
1234
+ },
1235
+ "128154": {
1236
+ "content": "<|reserved_special_token_146|>",
1237
+ "lstrip": false,
1238
+ "normalized": false,
1239
+ "rstrip": false,
1240
+ "single_word": false,
1241
+ "special": true
1242
+ },
1243
+ "128155": {
1244
+ "content": "<|reserved_special_token_147|>",
1245
+ "lstrip": false,
1246
+ "normalized": false,
1247
+ "rstrip": false,
1248
+ "single_word": false,
1249
+ "special": true
1250
+ },
1251
+ "128156": {
1252
+ "content": "<|reserved_special_token_148|>",
1253
+ "lstrip": false,
1254
+ "normalized": false,
1255
+ "rstrip": false,
1256
+ "single_word": false,
1257
+ "special": true
1258
+ },
1259
+ "128157": {
1260
+ "content": "<|reserved_special_token_149|>",
1261
+ "lstrip": false,
1262
+ "normalized": false,
1263
+ "rstrip": false,
1264
+ "single_word": false,
1265
+ "special": true
1266
+ },
1267
+ "128158": {
1268
+ "content": "<|reserved_special_token_150|>",
1269
+ "lstrip": false,
1270
+ "normalized": false,
1271
+ "rstrip": false,
1272
+ "single_word": false,
1273
+ "special": true
1274
+ },
1275
+ "128159": {
1276
+ "content": "<|reserved_special_token_151|>",
1277
+ "lstrip": false,
1278
+ "normalized": false,
1279
+ "rstrip": false,
1280
+ "single_word": false,
1281
+ "special": true
1282
+ },
1283
+ "128160": {
1284
+ "content": "<|reserved_special_token_152|>",
1285
+ "lstrip": false,
1286
+ "normalized": false,
1287
+ "rstrip": false,
1288
+ "single_word": false,
1289
+ "special": true
1290
+ },
1291
+ "128161": {
1292
+ "content": "<|reserved_special_token_153|>",
1293
+ "lstrip": false,
1294
+ "normalized": false,
1295
+ "rstrip": false,
1296
+ "single_word": false,
1297
+ "special": true
1298
+ },
1299
+ "128162": {
1300
+ "content": "<|reserved_special_token_154|>",
1301
+ "lstrip": false,
1302
+ "normalized": false,
1303
+ "rstrip": false,
1304
+ "single_word": false,
1305
+ "special": true
1306
+ },
1307
+ "128163": {
1308
+ "content": "<|reserved_special_token_155|>",
1309
+ "lstrip": false,
1310
+ "normalized": false,
1311
+ "rstrip": false,
1312
+ "single_word": false,
1313
+ "special": true
1314
+ },
1315
+ "128164": {
1316
+ "content": "<|reserved_special_token_156|>",
1317
+ "lstrip": false,
1318
+ "normalized": false,
1319
+ "rstrip": false,
1320
+ "single_word": false,
1321
+ "special": true
1322
+ },
1323
+ "128165": {
1324
+ "content": "<|reserved_special_token_157|>",
1325
+ "lstrip": false,
1326
+ "normalized": false,
1327
+ "rstrip": false,
1328
+ "single_word": false,
1329
+ "special": true
1330
+ },
1331
+ "128166": {
1332
+ "content": "<|reserved_special_token_158|>",
1333
+ "lstrip": false,
1334
+ "normalized": false,
1335
+ "rstrip": false,
1336
+ "single_word": false,
1337
+ "special": true
1338
+ },
1339
+ "128167": {
1340
+ "content": "<|reserved_special_token_159|>",
1341
+ "lstrip": false,
1342
+ "normalized": false,
1343
+ "rstrip": false,
1344
+ "single_word": false,
1345
+ "special": true
1346
+ },
1347
+ "128168": {
1348
+ "content": "<|reserved_special_token_160|>",
1349
+ "lstrip": false,
1350
+ "normalized": false,
1351
+ "rstrip": false,
1352
+ "single_word": false,
1353
+ "special": true
1354
+ },
1355
+ "128169": {
1356
+ "content": "<|reserved_special_token_161|>",
1357
+ "lstrip": false,
1358
+ "normalized": false,
1359
+ "rstrip": false,
1360
+ "single_word": false,
1361
+ "special": true
1362
+ },
1363
+ "128170": {
1364
+ "content": "<|reserved_special_token_162|>",
1365
+ "lstrip": false,
1366
+ "normalized": false,
1367
+ "rstrip": false,
1368
+ "single_word": false,
1369
+ "special": true
1370
+ },
1371
+ "128171": {
1372
+ "content": "<|reserved_special_token_163|>",
1373
+ "lstrip": false,
1374
+ "normalized": false,
1375
+ "rstrip": false,
1376
+ "single_word": false,
1377
+ "special": true
1378
+ },
1379
+ "128172": {
1380
+ "content": "<|reserved_special_token_164|>",
1381
+ "lstrip": false,
1382
+ "normalized": false,
1383
+ "rstrip": false,
1384
+ "single_word": false,
1385
+ "special": true
1386
+ },
1387
+ "128173": {
1388
+ "content": "<|reserved_special_token_165|>",
1389
+ "lstrip": false,
1390
+ "normalized": false,
1391
+ "rstrip": false,
1392
+ "single_word": false,
1393
+ "special": true
1394
+ },
1395
+ "128174": {
1396
+ "content": "<|reserved_special_token_166|>",
1397
+ "lstrip": false,
1398
+ "normalized": false,
1399
+ "rstrip": false,
1400
+ "single_word": false,
1401
+ "special": true
1402
+ },
1403
+ "128175": {
1404
+ "content": "<|reserved_special_token_167|>",
1405
+ "lstrip": false,
1406
+ "normalized": false,
1407
+ "rstrip": false,
1408
+ "single_word": false,
1409
+ "special": true
1410
+ },
1411
+ "128176": {
1412
+ "content": "<|reserved_special_token_168|>",
1413
+ "lstrip": false,
1414
+ "normalized": false,
1415
+ "rstrip": false,
1416
+ "single_word": false,
1417
+ "special": true
1418
+ },
1419
+ "128177": {
1420
+ "content": "<|reserved_special_token_169|>",
1421
+ "lstrip": false,
1422
+ "normalized": false,
1423
+ "rstrip": false,
1424
+ "single_word": false,
1425
+ "special": true
1426
+ },
1427
+ "128178": {
1428
+ "content": "<|reserved_special_token_170|>",
1429
+ "lstrip": false,
1430
+ "normalized": false,
1431
+ "rstrip": false,
1432
+ "single_word": false,
1433
+ "special": true
1434
+ },
1435
+ "128179": {
1436
+ "content": "<|reserved_special_token_171|>",
1437
+ "lstrip": false,
1438
+ "normalized": false,
1439
+ "rstrip": false,
1440
+ "single_word": false,
1441
+ "special": true
1442
+ },
1443
+ "128180": {
1444
+ "content": "<|reserved_special_token_172|>",
1445
+ "lstrip": false,
1446
+ "normalized": false,
1447
+ "rstrip": false,
1448
+ "single_word": false,
1449
+ "special": true
1450
+ },
1451
+ "128181": {
1452
+ "content": "<|reserved_special_token_173|>",
1453
+ "lstrip": false,
1454
+ "normalized": false,
1455
+ "rstrip": false,
1456
+ "single_word": false,
1457
+ "special": true
1458
+ },
1459
+ "128182": {
1460
+ "content": "<|reserved_special_token_174|>",
1461
+ "lstrip": false,
1462
+ "normalized": false,
1463
+ "rstrip": false,
1464
+ "single_word": false,
1465
+ "special": true
1466
+ },
1467
+ "128183": {
1468
+ "content": "<|reserved_special_token_175|>",
1469
+ "lstrip": false,
1470
+ "normalized": false,
1471
+ "rstrip": false,
1472
+ "single_word": false,
1473
+ "special": true
1474
+ },
1475
+ "128184": {
1476
+ "content": "<|reserved_special_token_176|>",
1477
+ "lstrip": false,
1478
+ "normalized": false,
1479
+ "rstrip": false,
1480
+ "single_word": false,
1481
+ "special": true
1482
+ },
1483
+ "128185": {
1484
+ "content": "<|reserved_special_token_177|>",
1485
+ "lstrip": false,
1486
+ "normalized": false,
1487
+ "rstrip": false,
1488
+ "single_word": false,
1489
+ "special": true
1490
+ },
1491
+ "128186": {
1492
+ "content": "<|reserved_special_token_178|>",
1493
+ "lstrip": false,
1494
+ "normalized": false,
1495
+ "rstrip": false,
1496
+ "single_word": false,
1497
+ "special": true
1498
+ },
1499
+ "128187": {
1500
+ "content": "<|reserved_special_token_179|>",
1501
+ "lstrip": false,
1502
+ "normalized": false,
1503
+ "rstrip": false,
1504
+ "single_word": false,
1505
+ "special": true
1506
+ },
1507
+ "128188": {
1508
+ "content": "<|reserved_special_token_180|>",
1509
+ "lstrip": false,
1510
+ "normalized": false,
1511
+ "rstrip": false,
1512
+ "single_word": false,
1513
+ "special": true
1514
+ },
1515
+ "128189": {
1516
+ "content": "<|reserved_special_token_181|>",
1517
+ "lstrip": false,
1518
+ "normalized": false,
1519
+ "rstrip": false,
1520
+ "single_word": false,
1521
+ "special": true
1522
+ },
1523
+ "128190": {
1524
+ "content": "<|reserved_special_token_182|>",
1525
+ "lstrip": false,
1526
+ "normalized": false,
1527
+ "rstrip": false,
1528
+ "single_word": false,
1529
+ "special": true
1530
+ },
1531
+ "128191": {
1532
+ "content": "<|reserved_special_token_183|>",
1533
+ "lstrip": false,
1534
+ "normalized": false,
1535
+ "rstrip": false,
1536
+ "single_word": false,
1537
+ "special": true
1538
+ },
1539
+ "128192": {
1540
+ "content": "<|reserved_special_token_184|>",
1541
+ "lstrip": false,
1542
+ "normalized": false,
1543
+ "rstrip": false,
1544
+ "single_word": false,
1545
+ "special": true
1546
+ },
1547
+ "128193": {
1548
+ "content": "<|reserved_special_token_185|>",
1549
+ "lstrip": false,
1550
+ "normalized": false,
1551
+ "rstrip": false,
1552
+ "single_word": false,
1553
+ "special": true
1554
+ },
1555
+ "128194": {
1556
+ "content": "<|reserved_special_token_186|>",
1557
+ "lstrip": false,
1558
+ "normalized": false,
1559
+ "rstrip": false,
1560
+ "single_word": false,
1561
+ "special": true
1562
+ },
1563
+ "128195": {
1564
+ "content": "<|reserved_special_token_187|>",
1565
+ "lstrip": false,
1566
+ "normalized": false,
1567
+ "rstrip": false,
1568
+ "single_word": false,
1569
+ "special": true
1570
+ },
1571
+ "128196": {
1572
+ "content": "<|reserved_special_token_188|>",
1573
+ "lstrip": false,
1574
+ "normalized": false,
1575
+ "rstrip": false,
1576
+ "single_word": false,
1577
+ "special": true
1578
+ },
1579
+ "128197": {
1580
+ "content": "<|reserved_special_token_189|>",
1581
+ "lstrip": false,
1582
+ "normalized": false,
1583
+ "rstrip": false,
1584
+ "single_word": false,
1585
+ "special": true
1586
+ },
1587
+ "128198": {
1588
+ "content": "<|reserved_special_token_190|>",
1589
+ "lstrip": false,
1590
+ "normalized": false,
1591
+ "rstrip": false,
1592
+ "single_word": false,
1593
+ "special": true
1594
+ },
1595
+ "128199": {
1596
+ "content": "<|reserved_special_token_191|>",
1597
+ "lstrip": false,
1598
+ "normalized": false,
1599
+ "rstrip": false,
1600
+ "single_word": false,
1601
+ "special": true
1602
+ },
1603
+ "128200": {
1604
+ "content": "<|reserved_special_token_192|>",
1605
+ "lstrip": false,
1606
+ "normalized": false,
1607
+ "rstrip": false,
1608
+ "single_word": false,
1609
+ "special": true
1610
+ },
1611
+ "128201": {
1612
+ "content": "<|reserved_special_token_193|>",
1613
+ "lstrip": false,
1614
+ "normalized": false,
1615
+ "rstrip": false,
1616
+ "single_word": false,
1617
+ "special": true
1618
+ },
1619
+ "128202": {
1620
+ "content": "<|reserved_special_token_194|>",
1621
+ "lstrip": false,
1622
+ "normalized": false,
1623
+ "rstrip": false,
1624
+ "single_word": false,
1625
+ "special": true
1626
+ },
1627
+ "128203": {
1628
+ "content": "<|reserved_special_token_195|>",
1629
+ "lstrip": false,
1630
+ "normalized": false,
1631
+ "rstrip": false,
1632
+ "single_word": false,
1633
+ "special": true
1634
+ },
1635
+ "128204": {
1636
+ "content": "<|reserved_special_token_196|>",
1637
+ "lstrip": false,
1638
+ "normalized": false,
1639
+ "rstrip": false,
1640
+ "single_word": false,
1641
+ "special": true
1642
+ },
1643
+ "128205": {
1644
+ "content": "<|reserved_special_token_197|>",
1645
+ "lstrip": false,
1646
+ "normalized": false,
1647
+ "rstrip": false,
1648
+ "single_word": false,
1649
+ "special": true
1650
+ },
1651
+ "128206": {
1652
+ "content": "<|reserved_special_token_198|>",
1653
+ "lstrip": false,
1654
+ "normalized": false,
1655
+ "rstrip": false,
1656
+ "single_word": false,
1657
+ "special": true
1658
+ },
1659
+ "128207": {
1660
+ "content": "<|reserved_special_token_199|>",
1661
+ "lstrip": false,
1662
+ "normalized": false,
1663
+ "rstrip": false,
1664
+ "single_word": false,
1665
+ "special": true
1666
+ },
1667
+ "128208": {
1668
+ "content": "<|reserved_special_token_200|>",
1669
+ "lstrip": false,
1670
+ "normalized": false,
1671
+ "rstrip": false,
1672
+ "single_word": false,
1673
+ "special": true
1674
+ },
1675
+ "128209": {
1676
+ "content": "<|reserved_special_token_201|>",
1677
+ "lstrip": false,
1678
+ "normalized": false,
1679
+ "rstrip": false,
1680
+ "single_word": false,
1681
+ "special": true
1682
+ },
1683
+ "128210": {
1684
+ "content": "<|reserved_special_token_202|>",
1685
+ "lstrip": false,
1686
+ "normalized": false,
1687
+ "rstrip": false,
1688
+ "single_word": false,
1689
+ "special": true
1690
+ },
1691
+ "128211": {
1692
+ "content": "<|reserved_special_token_203|>",
1693
+ "lstrip": false,
1694
+ "normalized": false,
1695
+ "rstrip": false,
1696
+ "single_word": false,
1697
+ "special": true
1698
+ },
1699
+ "128212": {
1700
+ "content": "<|reserved_special_token_204|>",
1701
+ "lstrip": false,
1702
+ "normalized": false,
1703
+ "rstrip": false,
1704
+ "single_word": false,
1705
+ "special": true
1706
+ },
1707
+ "128213": {
1708
+ "content": "<|reserved_special_token_205|>",
1709
+ "lstrip": false,
1710
+ "normalized": false,
1711
+ "rstrip": false,
1712
+ "single_word": false,
1713
+ "special": true
1714
+ },
1715
+ "128214": {
1716
+ "content": "<|reserved_special_token_206|>",
1717
+ "lstrip": false,
1718
+ "normalized": false,
1719
+ "rstrip": false,
1720
+ "single_word": false,
1721
+ "special": true
1722
+ },
1723
+ "128215": {
1724
+ "content": "<|reserved_special_token_207|>",
1725
+ "lstrip": false,
1726
+ "normalized": false,
1727
+ "rstrip": false,
1728
+ "single_word": false,
1729
+ "special": true
1730
+ },
1731
+ "128216": {
1732
+ "content": "<|reserved_special_token_208|>",
1733
+ "lstrip": false,
1734
+ "normalized": false,
1735
+ "rstrip": false,
1736
+ "single_word": false,
1737
+ "special": true
1738
+ },
1739
+ "128217": {
1740
+ "content": "<|reserved_special_token_209|>",
1741
+ "lstrip": false,
1742
+ "normalized": false,
1743
+ "rstrip": false,
1744
+ "single_word": false,
1745
+ "special": true
1746
+ },
1747
+ "128218": {
1748
+ "content": "<|reserved_special_token_210|>",
1749
+ "lstrip": false,
1750
+ "normalized": false,
1751
+ "rstrip": false,
1752
+ "single_word": false,
1753
+ "special": true
1754
+ },
1755
+ "128219": {
1756
+ "content": "<|reserved_special_token_211|>",
1757
+ "lstrip": false,
1758
+ "normalized": false,
1759
+ "rstrip": false,
1760
+ "single_word": false,
1761
+ "special": true
1762
+ },
1763
+ "128220": {
1764
+ "content": "<|reserved_special_token_212|>",
1765
+ "lstrip": false,
1766
+ "normalized": false,
1767
+ "rstrip": false,
1768
+ "single_word": false,
1769
+ "special": true
1770
+ },
1771
+ "128221": {
1772
+ "content": "<|reserved_special_token_213|>",
1773
+ "lstrip": false,
1774
+ "normalized": false,
1775
+ "rstrip": false,
1776
+ "single_word": false,
1777
+ "special": true
1778
+ },
1779
+ "128222": {
1780
+ "content": "<|reserved_special_token_214|>",
1781
+ "lstrip": false,
1782
+ "normalized": false,
1783
+ "rstrip": false,
1784
+ "single_word": false,
1785
+ "special": true
1786
+ },
1787
+ "128223": {
1788
+ "content": "<|reserved_special_token_215|>",
1789
+ "lstrip": false,
1790
+ "normalized": false,
1791
+ "rstrip": false,
1792
+ "single_word": false,
1793
+ "special": true
1794
+ },
1795
+ "128224": {
1796
+ "content": "<|reserved_special_token_216|>",
1797
+ "lstrip": false,
1798
+ "normalized": false,
1799
+ "rstrip": false,
1800
+ "single_word": false,
1801
+ "special": true
1802
+ },
1803
+ "128225": {
1804
+ "content": "<|reserved_special_token_217|>",
1805
+ "lstrip": false,
1806
+ "normalized": false,
1807
+ "rstrip": false,
1808
+ "single_word": false,
1809
+ "special": true
1810
+ },
1811
+ "128226": {
1812
+ "content": "<|reserved_special_token_218|>",
1813
+ "lstrip": false,
1814
+ "normalized": false,
1815
+ "rstrip": false,
1816
+ "single_word": false,
1817
+ "special": true
1818
+ },
1819
+ "128227": {
1820
+ "content": "<|reserved_special_token_219|>",
1821
+ "lstrip": false,
1822
+ "normalized": false,
1823
+ "rstrip": false,
1824
+ "single_word": false,
1825
+ "special": true
1826
+ },
1827
+ "128228": {
1828
+ "content": "<|reserved_special_token_220|>",
1829
+ "lstrip": false,
1830
+ "normalized": false,
1831
+ "rstrip": false,
1832
+ "single_word": false,
1833
+ "special": true
1834
+ },
1835
+ "128229": {
1836
+ "content": "<|reserved_special_token_221|>",
1837
+ "lstrip": false,
1838
+ "normalized": false,
1839
+ "rstrip": false,
1840
+ "single_word": false,
1841
+ "special": true
1842
+ },
1843
+ "128230": {
1844
+ "content": "<|reserved_special_token_222|>",
1845
+ "lstrip": false,
1846
+ "normalized": false,
1847
+ "rstrip": false,
1848
+ "single_word": false,
1849
+ "special": true
1850
+ },
1851
+ "128231": {
1852
+ "content": "<|reserved_special_token_223|>",
1853
+ "lstrip": false,
1854
+ "normalized": false,
1855
+ "rstrip": false,
1856
+ "single_word": false,
1857
+ "special": true
1858
+ },
1859
+ "128232": {
1860
+ "content": "<|reserved_special_token_224|>",
1861
+ "lstrip": false,
1862
+ "normalized": false,
1863
+ "rstrip": false,
1864
+ "single_word": false,
1865
+ "special": true
1866
+ },
1867
+ "128233": {
1868
+ "content": "<|reserved_special_token_225|>",
1869
+ "lstrip": false,
1870
+ "normalized": false,
1871
+ "rstrip": false,
1872
+ "single_word": false,
1873
+ "special": true
1874
+ },
1875
+ "128234": {
1876
+ "content": "<|reserved_special_token_226|>",
1877
+ "lstrip": false,
1878
+ "normalized": false,
1879
+ "rstrip": false,
1880
+ "single_word": false,
1881
+ "special": true
1882
+ },
1883
+ "128235": {
1884
+ "content": "<|reserved_special_token_227|>",
1885
+ "lstrip": false,
1886
+ "normalized": false,
1887
+ "rstrip": false,
1888
+ "single_word": false,
1889
+ "special": true
1890
+ },
1891
+ "128236": {
1892
+ "content": "<|reserved_special_token_228|>",
1893
+ "lstrip": false,
1894
+ "normalized": false,
1895
+ "rstrip": false,
1896
+ "single_word": false,
1897
+ "special": true
1898
+ },
1899
+ "128237": {
1900
+ "content": "<|reserved_special_token_229|>",
1901
+ "lstrip": false,
1902
+ "normalized": false,
1903
+ "rstrip": false,
1904
+ "single_word": false,
1905
+ "special": true
1906
+ },
1907
+ "128238": {
1908
+ "content": "<|reserved_special_token_230|>",
1909
+ "lstrip": false,
1910
+ "normalized": false,
1911
+ "rstrip": false,
1912
+ "single_word": false,
1913
+ "special": true
1914
+ },
1915
+ "128239": {
1916
+ "content": "<|reserved_special_token_231|>",
1917
+ "lstrip": false,
1918
+ "normalized": false,
1919
+ "rstrip": false,
1920
+ "single_word": false,
1921
+ "special": true
1922
+ },
1923
+ "128240": {
1924
+ "content": "<|reserved_special_token_232|>",
1925
+ "lstrip": false,
1926
+ "normalized": false,
1927
+ "rstrip": false,
1928
+ "single_word": false,
1929
+ "special": true
1930
+ },
1931
+ "128241": {
1932
+ "content": "<|reserved_special_token_233|>",
1933
+ "lstrip": false,
1934
+ "normalized": false,
1935
+ "rstrip": false,
1936
+ "single_word": false,
1937
+ "special": true
1938
+ },
1939
+ "128242": {
1940
+ "content": "<|reserved_special_token_234|>",
1941
+ "lstrip": false,
1942
+ "normalized": false,
1943
+ "rstrip": false,
1944
+ "single_word": false,
1945
+ "special": true
1946
+ },
1947
+ "128243": {
1948
+ "content": "<|reserved_special_token_235|>",
1949
+ "lstrip": false,
1950
+ "normalized": false,
1951
+ "rstrip": false,
1952
+ "single_word": false,
1953
+ "special": true
1954
+ },
1955
+ "128244": {
1956
+ "content": "<|reserved_special_token_236|>",
1957
+ "lstrip": false,
1958
+ "normalized": false,
1959
+ "rstrip": false,
1960
+ "single_word": false,
1961
+ "special": true
1962
+ },
1963
+ "128245": {
1964
+ "content": "<|reserved_special_token_237|>",
1965
+ "lstrip": false,
1966
+ "normalized": false,
1967
+ "rstrip": false,
1968
+ "single_word": false,
1969
+ "special": true
1970
+ },
1971
+ "128246": {
1972
+ "content": "<|reserved_special_token_238|>",
1973
+ "lstrip": false,
1974
+ "normalized": false,
1975
+ "rstrip": false,
1976
+ "single_word": false,
1977
+ "special": true
1978
+ },
1979
+ "128247": {
1980
+ "content": "<|reserved_special_token_239|>",
1981
+ "lstrip": false,
1982
+ "normalized": false,
1983
+ "rstrip": false,
1984
+ "single_word": false,
1985
+ "special": true
1986
+ },
1987
+ "128248": {
1988
+ "content": "<|reserved_special_token_240|>",
1989
+ "lstrip": false,
1990
+ "normalized": false,
1991
+ "rstrip": false,
1992
+ "single_word": false,
1993
+ "special": true
1994
+ },
1995
+ "128249": {
1996
+ "content": "<|reserved_special_token_241|>",
1997
+ "lstrip": false,
1998
+ "normalized": false,
1999
+ "rstrip": false,
2000
+ "single_word": false,
2001
+ "special": true
2002
+ },
2003
+ "128250": {
2004
+ "content": "<|reserved_special_token_242|>",
2005
+ "lstrip": false,
2006
+ "normalized": false,
2007
+ "rstrip": false,
2008
+ "single_word": false,
2009
+ "special": true
2010
+ },
2011
+ "128251": {
2012
+ "content": "<|reserved_special_token_243|>",
2013
+ "lstrip": false,
2014
+ "normalized": false,
2015
+ "rstrip": false,
2016
+ "single_word": false,
2017
+ "special": true
2018
+ },
2019
+ "128252": {
2020
+ "content": "<|reserved_special_token_244|>",
2021
+ "lstrip": false,
2022
+ "normalized": false,
2023
+ "rstrip": false,
2024
+ "single_word": false,
2025
+ "special": true
2026
+ },
2027
+ "128253": {
2028
+ "content": "<|reserved_special_token_245|>",
2029
+ "lstrip": false,
2030
+ "normalized": false,
2031
+ "rstrip": false,
2032
+ "single_word": false,
2033
+ "special": true
2034
+ },
2035
+ "128254": {
2036
+ "content": "<|reserved_special_token_246|>",
2037
+ "lstrip": false,
2038
+ "normalized": false,
2039
+ "rstrip": false,
2040
+ "single_word": false,
2041
+ "special": true
2042
+ },
2043
+ "128255": {
2044
+ "content": "<|reserved_special_token_247|>",
2045
+ "lstrip": false,
2046
+ "normalized": false,
2047
+ "rstrip": false,
2048
+ "single_word": false,
2049
+ "special": true
2050
+ }
2051
+ },
2052
+ "bos_token": "<|begin_of_text|>",
2053
+ "chat_template": "{{ '<|begin_of_text|>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %}",
2054
+ "clean_up_tokenization_spaces": true,
2055
+ "eos_token": "<|eot_id|>",
2056
+ "model_input_names": [
2057
+ "input_ids",
2058
+ "attention_mask"
2059
+ ],
2060
+ "model_max_length": 131072,
2061
+ "pad_token": "<|eot_id|>",
2062
+ "padding_side": "right",
2063
+ "split_special_tokens": false,
2064
+ "tokenizer_class": "PreTrainedTokenizerFast"
2065
+ }
trainer_state.json ADDED
@@ -0,0 +1,3743 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.0,
5
+ "eval_steps": 500.0,
6
+ "global_step": 26534,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.003768749528906309,
13
+ "grad_norm": 0.2906060218811035,
14
+ "learning_rate": 9.999912386093533e-05,
15
+ "loss": 0.5663,
16
+ "step": 50
17
+ },
18
+ {
19
+ "epoch": 0.007537499057812618,
20
+ "grad_norm": 0.34499412775039673,
21
+ "learning_rate": 9.999649547444612e-05,
22
+ "loss": 0.5446,
23
+ "step": 100
24
+ },
25
+ {
26
+ "epoch": 0.011306248586718927,
27
+ "grad_norm": 0.3074498772621155,
28
+ "learning_rate": 9.999211493264562e-05,
29
+ "loss": 0.5425,
30
+ "step": 150
31
+ },
32
+ {
33
+ "epoch": 0.015074998115625236,
34
+ "grad_norm": 0.29462873935699463,
35
+ "learning_rate": 9.998598238905239e-05,
36
+ "loss": 0.5447,
37
+ "step": 200
38
+ },
39
+ {
40
+ "epoch": 0.018843747644531544,
41
+ "grad_norm": 0.28769969940185547,
42
+ "learning_rate": 9.99780980585849e-05,
43
+ "loss": 0.5443,
44
+ "step": 250
45
+ },
46
+ {
47
+ "epoch": 0.022612497173437853,
48
+ "grad_norm": 0.2771708369255066,
49
+ "learning_rate": 9.996846221755392e-05,
50
+ "loss": 0.543,
51
+ "step": 300
52
+ },
53
+ {
54
+ "epoch": 0.026381246702344163,
55
+ "grad_norm": 0.27388525009155273,
56
+ "learning_rate": 9.995707520365293e-05,
57
+ "loss": 0.5437,
58
+ "step": 350
59
+ },
60
+ {
61
+ "epoch": 0.030149996231250472,
62
+ "grad_norm": 0.29543155431747437,
63
+ "learning_rate": 9.994393741594623e-05,
64
+ "loss": 0.5428,
65
+ "step": 400
66
+ },
67
+ {
68
+ "epoch": 0.03391874576015678,
69
+ "grad_norm": 0.2794398069381714,
70
+ "learning_rate": 9.9929049314855e-05,
71
+ "loss": 0.5404,
72
+ "step": 450
73
+ },
74
+ {
75
+ "epoch": 0.03768749528906309,
76
+ "grad_norm": 0.2607417702674866,
77
+ "learning_rate": 9.99124114221411e-05,
78
+ "loss": 0.5451,
79
+ "step": 500
80
+ },
81
+ {
82
+ "epoch": 0.0414562448179694,
83
+ "grad_norm": 0.24695731699466705,
84
+ "learning_rate": 9.989402432088884e-05,
85
+ "loss": 0.5435,
86
+ "step": 550
87
+ },
88
+ {
89
+ "epoch": 0.045224994346875706,
90
+ "grad_norm": 0.2468685656785965,
91
+ "learning_rate": 9.987388865548454e-05,
92
+ "loss": 0.54,
93
+ "step": 600
94
+ },
95
+ {
96
+ "epoch": 0.048993743875782016,
97
+ "grad_norm": 0.25657713413238525,
98
+ "learning_rate": 9.985200513159391e-05,
99
+ "loss": 0.5423,
100
+ "step": 650
101
+ },
102
+ {
103
+ "epoch": 0.052762493404688325,
104
+ "grad_norm": 0.2581389546394348,
105
+ "learning_rate": 9.982837451613738e-05,
106
+ "loss": 0.5389,
107
+ "step": 700
108
+ },
109
+ {
110
+ "epoch": 0.056531242933594635,
111
+ "grad_norm": 0.25357818603515625,
112
+ "learning_rate": 9.980299763726313e-05,
113
+ "loss": 0.5383,
114
+ "step": 750
115
+ },
116
+ {
117
+ "epoch": 0.060299992462500944,
118
+ "grad_norm": 0.27078768610954285,
119
+ "learning_rate": 9.977587538431816e-05,
120
+ "loss": 0.5385,
121
+ "step": 800
122
+ },
123
+ {
124
+ "epoch": 0.06406874199140725,
125
+ "grad_norm": 0.23457016050815582,
126
+ "learning_rate": 9.974700870781712e-05,
127
+ "loss": 0.5384,
128
+ "step": 850
129
+ },
130
+ {
131
+ "epoch": 0.06783749152031356,
132
+ "grad_norm": 0.24491585791110992,
133
+ "learning_rate": 9.971639861940889e-05,
134
+ "loss": 0.5362,
135
+ "step": 900
136
+ },
137
+ {
138
+ "epoch": 0.07160624104921987,
139
+ "grad_norm": 0.24196478724479675,
140
+ "learning_rate": 9.968404619184125e-05,
141
+ "loss": 0.5384,
142
+ "step": 950
143
+ },
144
+ {
145
+ "epoch": 0.07537499057812617,
146
+ "grad_norm": 0.24577759206295013,
147
+ "learning_rate": 9.964995255892323e-05,
148
+ "loss": 0.536,
149
+ "step": 1000
150
+ },
151
+ {
152
+ "epoch": 0.07914374010703248,
153
+ "grad_norm": 0.23471598327159882,
154
+ "learning_rate": 9.961411891548538e-05,
155
+ "loss": 0.536,
156
+ "step": 1050
157
+ },
158
+ {
159
+ "epoch": 0.0829124896359388,
160
+ "grad_norm": 0.2284562885761261,
161
+ "learning_rate": 9.957654651733788e-05,
162
+ "loss": 0.5347,
163
+ "step": 1100
164
+ },
165
+ {
166
+ "epoch": 0.0866812391648451,
167
+ "grad_norm": 0.267677366733551,
168
+ "learning_rate": 9.953723668122656e-05,
169
+ "loss": 0.5351,
170
+ "step": 1150
171
+ },
172
+ {
173
+ "epoch": 0.09044998869375141,
174
+ "grad_norm": 0.23843041062355042,
175
+ "learning_rate": 9.949619078478677e-05,
176
+ "loss": 0.5331,
177
+ "step": 1200
178
+ },
179
+ {
180
+ "epoch": 0.09421873822265772,
181
+ "grad_norm": 0.2461230456829071,
182
+ "learning_rate": 9.945341026649501e-05,
183
+ "loss": 0.5344,
184
+ "step": 1250
185
+ },
186
+ {
187
+ "epoch": 0.09798748775156403,
188
+ "grad_norm": 0.22946181893348694,
189
+ "learning_rate": 9.940889662561864e-05,
190
+ "loss": 0.5328,
191
+ "step": 1300
192
+ },
193
+ {
194
+ "epoch": 0.10175623728047034,
195
+ "grad_norm": 0.22417663037776947,
196
+ "learning_rate": 9.93626514221632e-05,
197
+ "loss": 0.5309,
198
+ "step": 1350
199
+ },
200
+ {
201
+ "epoch": 0.10552498680937665,
202
+ "grad_norm": 0.24278171360492706,
203
+ "learning_rate": 9.931467627681792e-05,
204
+ "loss": 0.5317,
205
+ "step": 1400
206
+ },
207
+ {
208
+ "epoch": 0.10929373633828296,
209
+ "grad_norm": 0.25257062911987305,
210
+ "learning_rate": 9.926497287089871e-05,
211
+ "loss": 0.5313,
212
+ "step": 1450
213
+ },
214
+ {
215
+ "epoch": 0.11306248586718927,
216
+ "grad_norm": 0.2244357168674469,
217
+ "learning_rate": 9.921354294628944e-05,
218
+ "loss": 0.5318,
219
+ "step": 1500
220
+ },
221
+ {
222
+ "epoch": 0.11683123539609558,
223
+ "grad_norm": 0.22987781465053558,
224
+ "learning_rate": 9.916038830538071e-05,
225
+ "loss": 0.53,
226
+ "step": 1550
227
+ },
228
+ {
229
+ "epoch": 0.12059998492500189,
230
+ "grad_norm": 0.22749993205070496,
231
+ "learning_rate": 9.910551081100684e-05,
232
+ "loss": 0.5288,
233
+ "step": 1600
234
+ },
235
+ {
236
+ "epoch": 0.1243687344539082,
237
+ "grad_norm": 0.23521047830581665,
238
+ "learning_rate": 9.904891238638048e-05,
239
+ "loss": 0.5286,
240
+ "step": 1650
241
+ },
242
+ {
243
+ "epoch": 0.1281374839828145,
244
+ "grad_norm": 0.23071640729904175,
245
+ "learning_rate": 9.899059501502526e-05,
246
+ "loss": 0.5266,
247
+ "step": 1700
248
+ },
249
+ {
250
+ "epoch": 0.13190623351172082,
251
+ "grad_norm": 0.21863527595996857,
252
+ "learning_rate": 9.893056074070629e-05,
253
+ "loss": 0.5271,
254
+ "step": 1750
255
+ },
256
+ {
257
+ "epoch": 0.1356749830406271,
258
+ "grad_norm": 0.21325771510601044,
259
+ "learning_rate": 9.886881166735846e-05,
260
+ "loss": 0.5236,
261
+ "step": 1800
262
+ },
263
+ {
264
+ "epoch": 0.13944373256953344,
265
+ "grad_norm": 0.2273271232843399,
266
+ "learning_rate": 9.88053499590128e-05,
267
+ "loss": 0.5246,
268
+ "step": 1850
269
+ },
270
+ {
271
+ "epoch": 0.14321248209843973,
272
+ "grad_norm": 0.22810806334018707,
273
+ "learning_rate": 9.874017783972058e-05,
274
+ "loss": 0.5257,
275
+ "step": 1900
276
+ },
277
+ {
278
+ "epoch": 0.14698123162734605,
279
+ "grad_norm": 0.21901291608810425,
280
+ "learning_rate": 9.867329759347538e-05,
281
+ "loss": 0.5245,
282
+ "step": 1950
283
+ },
284
+ {
285
+ "epoch": 0.15074998115625235,
286
+ "grad_norm": 0.2476150393486023,
287
+ "learning_rate": 9.860471156413309e-05,
288
+ "loss": 0.5247,
289
+ "step": 2000
290
+ },
291
+ {
292
+ "epoch": 0.15451873068515867,
293
+ "grad_norm": 0.22044631838798523,
294
+ "learning_rate": 9.853442215532964e-05,
295
+ "loss": 0.5259,
296
+ "step": 2050
297
+ },
298
+ {
299
+ "epoch": 0.15828748021406497,
300
+ "grad_norm": 0.20892663300037384,
301
+ "learning_rate": 9.846243183039694e-05,
302
+ "loss": 0.5239,
303
+ "step": 2100
304
+ },
305
+ {
306
+ "epoch": 0.1620562297429713,
307
+ "grad_norm": 0.2248477190732956,
308
+ "learning_rate": 9.83887431122764e-05,
309
+ "loss": 0.5215,
310
+ "step": 2150
311
+ },
312
+ {
313
+ "epoch": 0.1658249792718776,
314
+ "grad_norm": 0.21989433467388153,
315
+ "learning_rate": 9.831335858343064e-05,
316
+ "loss": 0.5235,
317
+ "step": 2200
318
+ },
319
+ {
320
+ "epoch": 0.1695937288007839,
321
+ "grad_norm": 0.2244018316268921,
322
+ "learning_rate": 9.823628088575285e-05,
323
+ "loss": 0.5226,
324
+ "step": 2250
325
+ },
326
+ {
327
+ "epoch": 0.1733624783296902,
328
+ "grad_norm": 0.21466925740242004,
329
+ "learning_rate": 9.815751272047434e-05,
330
+ "loss": 0.5215,
331
+ "step": 2300
332
+ },
333
+ {
334
+ "epoch": 0.17713122785859653,
335
+ "grad_norm": 0.21101678907871246,
336
+ "learning_rate": 9.807705684806975e-05,
337
+ "loss": 0.5207,
338
+ "step": 2350
339
+ },
340
+ {
341
+ "epoch": 0.18089997738750282,
342
+ "grad_norm": 0.21319390833377838,
343
+ "learning_rate": 9.79949160881604e-05,
344
+ "loss": 0.5198,
345
+ "step": 2400
346
+ },
347
+ {
348
+ "epoch": 0.18466872691640915,
349
+ "grad_norm": 0.2198377102613449,
350
+ "learning_rate": 9.791109331941543e-05,
351
+ "loss": 0.5233,
352
+ "step": 2450
353
+ },
354
+ {
355
+ "epoch": 0.18843747644531544,
356
+ "grad_norm": 0.33185112476348877,
357
+ "learning_rate": 9.782559147945094e-05,
358
+ "loss": 0.5203,
359
+ "step": 2500
360
+ },
361
+ {
362
+ "epoch": 0.19220622597422174,
363
+ "grad_norm": 0.22263136506080627,
364
+ "learning_rate": 9.773841356472701e-05,
365
+ "loss": 0.5193,
366
+ "step": 2550
367
+ },
368
+ {
369
+ "epoch": 0.19597497550312806,
370
+ "grad_norm": 0.23071548342704773,
371
+ "learning_rate": 9.76495626304427e-05,
372
+ "loss": 0.5184,
373
+ "step": 2600
374
+ },
375
+ {
376
+ "epoch": 0.19974372503203436,
377
+ "grad_norm": 0.21315498650074005,
378
+ "learning_rate": 9.755904179042898e-05,
379
+ "loss": 0.5175,
380
+ "step": 2650
381
+ },
382
+ {
383
+ "epoch": 0.20351247456094068,
384
+ "grad_norm": 0.2224321961402893,
385
+ "learning_rate": 9.746685421703961e-05,
386
+ "loss": 0.5185,
387
+ "step": 2700
388
+ },
389
+ {
390
+ "epoch": 0.20728122408984698,
391
+ "grad_norm": 0.20440195500850677,
392
+ "learning_rate": 9.737300314104e-05,
393
+ "loss": 0.5179,
394
+ "step": 2750
395
+ },
396
+ {
397
+ "epoch": 0.2110499736187533,
398
+ "grad_norm": 0.220822274684906,
399
+ "learning_rate": 9.727749185149388e-05,
400
+ "loss": 0.5159,
401
+ "step": 2800
402
+ },
403
+ {
404
+ "epoch": 0.2148187231476596,
405
+ "grad_norm": 0.2104455679655075,
406
+ "learning_rate": 9.718032369564814e-05,
407
+ "loss": 0.515,
408
+ "step": 2850
409
+ },
410
+ {
411
+ "epoch": 0.21858747267656592,
412
+ "grad_norm": 0.21458739042282104,
413
+ "learning_rate": 9.708150207881543e-05,
414
+ "loss": 0.5146,
415
+ "step": 2900
416
+ },
417
+ {
418
+ "epoch": 0.22235622220547221,
419
+ "grad_norm": 0.21516002714633942,
420
+ "learning_rate": 9.698103046425496e-05,
421
+ "loss": 0.5162,
422
+ "step": 2950
423
+ },
424
+ {
425
+ "epoch": 0.22612497173437854,
426
+ "grad_norm": 0.22671422362327576,
427
+ "learning_rate": 9.687891237305096e-05,
428
+ "loss": 0.5132,
429
+ "step": 3000
430
+ },
431
+ {
432
+ "epoch": 0.22989372126328483,
433
+ "grad_norm": 0.20153091847896576,
434
+ "learning_rate": 9.677515138398938e-05,
435
+ "loss": 0.5151,
436
+ "step": 3050
437
+ },
438
+ {
439
+ "epoch": 0.23366247079219116,
440
+ "grad_norm": 0.21473319828510284,
441
+ "learning_rate": 9.666975113343246e-05,
442
+ "loss": 0.5113,
443
+ "step": 3100
444
+ },
445
+ {
446
+ "epoch": 0.23743122032109745,
447
+ "grad_norm": 0.22579412162303925,
448
+ "learning_rate": 9.65627153151913e-05,
449
+ "loss": 0.5151,
450
+ "step": 3150
451
+ },
452
+ {
453
+ "epoch": 0.24119996985000378,
454
+ "grad_norm": 0.21873536705970764,
455
+ "learning_rate": 9.645404768039633e-05,
456
+ "loss": 0.5135,
457
+ "step": 3200
458
+ },
459
+ {
460
+ "epoch": 0.24496871937891007,
461
+ "grad_norm": 0.210534006357193,
462
+ "learning_rate": 9.634375203736597e-05,
463
+ "loss": 0.5125,
464
+ "step": 3250
465
+ },
466
+ {
467
+ "epoch": 0.2487374689078164,
468
+ "grad_norm": 0.2117706835269928,
469
+ "learning_rate": 9.623183225147308e-05,
470
+ "loss": 0.5138,
471
+ "step": 3300
472
+ },
473
+ {
474
+ "epoch": 0.2525062184367227,
475
+ "grad_norm": 0.20858237147331238,
476
+ "learning_rate": 9.611829224500953e-05,
477
+ "loss": 0.5107,
478
+ "step": 3350
479
+ },
480
+ {
481
+ "epoch": 0.256274967965629,
482
+ "grad_norm": 0.1984904557466507,
483
+ "learning_rate": 9.600313599704869e-05,
484
+ "loss": 0.5086,
485
+ "step": 3400
486
+ },
487
+ {
488
+ "epoch": 0.2600437174945353,
489
+ "grad_norm": 0.2197868824005127,
490
+ "learning_rate": 9.588636754330609e-05,
491
+ "loss": 0.511,
492
+ "step": 3450
493
+ },
494
+ {
495
+ "epoch": 0.26381246702344163,
496
+ "grad_norm": 0.21637748181819916,
497
+ "learning_rate": 9.576799097599786e-05,
498
+ "loss": 0.5113,
499
+ "step": 3500
500
+ },
501
+ {
502
+ "epoch": 0.26758121655234796,
503
+ "grad_norm": 0.2107384353876114,
504
+ "learning_rate": 9.56480104436974e-05,
505
+ "loss": 0.5133,
506
+ "step": 3550
507
+ },
508
+ {
509
+ "epoch": 0.2713499660812542,
510
+ "grad_norm": 0.21115067601203918,
511
+ "learning_rate": 9.552643015118998e-05,
512
+ "loss": 0.5102,
513
+ "step": 3600
514
+ },
515
+ {
516
+ "epoch": 0.27511871561016055,
517
+ "grad_norm": 0.2121039628982544,
518
+ "learning_rate": 9.540325435932536e-05,
519
+ "loss": 0.513,
520
+ "step": 3650
521
+ },
522
+ {
523
+ "epoch": 0.27888746513906687,
524
+ "grad_norm": 0.20221100747585297,
525
+ "learning_rate": 9.527848738486842e-05,
526
+ "loss": 0.5073,
527
+ "step": 3700
528
+ },
529
+ {
530
+ "epoch": 0.2826562146679732,
531
+ "grad_norm": 0.22263558208942413,
532
+ "learning_rate": 9.5152133600348e-05,
533
+ "loss": 0.509,
534
+ "step": 3750
535
+ },
536
+ {
537
+ "epoch": 0.28642496419687946,
538
+ "grad_norm": 0.1968437135219574,
539
+ "learning_rate": 9.502419743390357e-05,
540
+ "loss": 0.5089,
541
+ "step": 3800
542
+ },
543
+ {
544
+ "epoch": 0.2901937137257858,
545
+ "grad_norm": 0.21389682590961456,
546
+ "learning_rate": 9.489468336913003e-05,
547
+ "loss": 0.5095,
548
+ "step": 3850
549
+ },
550
+ {
551
+ "epoch": 0.2939624632546921,
552
+ "grad_norm": 0.20469701290130615,
553
+ "learning_rate": 9.476359594492068e-05,
554
+ "loss": 0.5099,
555
+ "step": 3900
556
+ },
557
+ {
558
+ "epoch": 0.2977312127835984,
559
+ "grad_norm": 0.20063601434230804,
560
+ "learning_rate": 9.463093975530803e-05,
561
+ "loss": 0.5054,
562
+ "step": 3950
563
+ },
564
+ {
565
+ "epoch": 0.3014999623125047,
566
+ "grad_norm": 0.20724664628505707,
567
+ "learning_rate": 9.449671944930288e-05,
568
+ "loss": 0.507,
569
+ "step": 4000
570
+ },
571
+ {
572
+ "epoch": 0.305268711841411,
573
+ "grad_norm": 0.19884862005710602,
574
+ "learning_rate": 9.436093973073135e-05,
575
+ "loss": 0.5063,
576
+ "step": 4050
577
+ },
578
+ {
579
+ "epoch": 0.30903746137031735,
580
+ "grad_norm": 0.21703405678272247,
581
+ "learning_rate": 9.422360535807009e-05,
582
+ "loss": 0.5083,
583
+ "step": 4100
584
+ },
585
+ {
586
+ "epoch": 0.3128062108992236,
587
+ "grad_norm": 0.20601245760917664,
588
+ "learning_rate": 9.408472114427943e-05,
589
+ "loss": 0.5049,
590
+ "step": 4150
591
+ },
592
+ {
593
+ "epoch": 0.31657496042812994,
594
+ "grad_norm": 0.20570008456707,
595
+ "learning_rate": 9.394429195663478e-05,
596
+ "loss": 0.5083,
597
+ "step": 4200
598
+ },
599
+ {
600
+ "epoch": 0.32034370995703626,
601
+ "grad_norm": 0.1968814581632614,
602
+ "learning_rate": 9.380232271655603e-05,
603
+ "loss": 0.5025,
604
+ "step": 4250
605
+ },
606
+ {
607
+ "epoch": 0.3241124594859426,
608
+ "grad_norm": 0.20559944212436676,
609
+ "learning_rate": 9.365881839943508e-05,
610
+ "loss": 0.5045,
611
+ "step": 4300
612
+ },
613
+ {
614
+ "epoch": 0.32788120901484885,
615
+ "grad_norm": 0.19153840839862823,
616
+ "learning_rate": 9.351378403446144e-05,
617
+ "loss": 0.5057,
618
+ "step": 4350
619
+ },
620
+ {
621
+ "epoch": 0.3316499585437552,
622
+ "grad_norm": 0.21196983754634857,
623
+ "learning_rate": 9.336722470444604e-05,
624
+ "loss": 0.5052,
625
+ "step": 4400
626
+ },
627
+ {
628
+ "epoch": 0.3354187080726615,
629
+ "grad_norm": 0.219445139169693,
630
+ "learning_rate": 9.321914554564302e-05,
631
+ "loss": 0.506,
632
+ "step": 4450
633
+ },
634
+ {
635
+ "epoch": 0.3391874576015678,
636
+ "grad_norm": 0.19869840145111084,
637
+ "learning_rate": 9.306955174756985e-05,
638
+ "loss": 0.5032,
639
+ "step": 4500
640
+ },
641
+ {
642
+ "epoch": 0.3429562071304741,
643
+ "grad_norm": 0.19720250368118286,
644
+ "learning_rate": 9.291844855282532e-05,
645
+ "loss": 0.5053,
646
+ "step": 4550
647
+ },
648
+ {
649
+ "epoch": 0.3467249566593804,
650
+ "grad_norm": 0.1970975250005722,
651
+ "learning_rate": 9.27658412569059e-05,
652
+ "loss": 0.5027,
653
+ "step": 4600
654
+ },
655
+ {
656
+ "epoch": 0.35049370618828674,
657
+ "grad_norm": 0.2193278670310974,
658
+ "learning_rate": 9.261173520802014e-05,
659
+ "loss": 0.5038,
660
+ "step": 4650
661
+ },
662
+ {
663
+ "epoch": 0.35426245571719306,
664
+ "grad_norm": 0.2011794000864029,
665
+ "learning_rate": 9.24561358069012e-05,
666
+ "loss": 0.5039,
667
+ "step": 4700
668
+ },
669
+ {
670
+ "epoch": 0.3580312052460993,
671
+ "grad_norm": 0.21378175914287567,
672
+ "learning_rate": 9.229904850661766e-05,
673
+ "loss": 0.5026,
674
+ "step": 4750
675
+ },
676
+ {
677
+ "epoch": 0.36179995477500565,
678
+ "grad_norm": 0.20413589477539062,
679
+ "learning_rate": 9.214047881238233e-05,
680
+ "loss": 0.5001,
681
+ "step": 4800
682
+ },
683
+ {
684
+ "epoch": 0.365568704303912,
685
+ "grad_norm": 0.20819853246212006,
686
+ "learning_rate": 9.19804322813593e-05,
687
+ "loss": 0.5011,
688
+ "step": 4850
689
+ },
690
+ {
691
+ "epoch": 0.3693374538328183,
692
+ "grad_norm": 0.20473329722881317,
693
+ "learning_rate": 9.181891452246937e-05,
694
+ "loss": 0.5018,
695
+ "step": 4900
696
+ },
697
+ {
698
+ "epoch": 0.37310620336172456,
699
+ "grad_norm": 0.1926797479391098,
700
+ "learning_rate": 9.165593119619321e-05,
701
+ "loss": 0.5021,
702
+ "step": 4950
703
+ },
704
+ {
705
+ "epoch": 0.3768749528906309,
706
+ "grad_norm": 0.19186194241046906,
707
+ "learning_rate": 9.149148801437321e-05,
708
+ "loss": 0.499,
709
+ "step": 5000
710
+ },
711
+ {
712
+ "epoch": 0.3806437024195372,
713
+ "grad_norm": 0.19856803119182587,
714
+ "learning_rate": 9.132559074001319e-05,
715
+ "loss": 0.5019,
716
+ "step": 5050
717
+ },
718
+ {
719
+ "epoch": 0.3844124519484435,
720
+ "grad_norm": 0.20318450033664703,
721
+ "learning_rate": 9.115824518707644e-05,
722
+ "loss": 0.5023,
723
+ "step": 5100
724
+ },
725
+ {
726
+ "epoch": 0.3881812014773498,
727
+ "grad_norm": 0.19259172677993774,
728
+ "learning_rate": 9.098945722028203e-05,
729
+ "loss": 0.4994,
730
+ "step": 5150
731
+ },
732
+ {
733
+ "epoch": 0.3919499510062561,
734
+ "grad_norm": 0.19474363327026367,
735
+ "learning_rate": 9.08192327548992e-05,
736
+ "loss": 0.4992,
737
+ "step": 5200
738
+ },
739
+ {
740
+ "epoch": 0.39571870053516245,
741
+ "grad_norm": 0.1978476643562317,
742
+ "learning_rate": 9.064757775654011e-05,
743
+ "loss": 0.4975,
744
+ "step": 5250
745
+ },
746
+ {
747
+ "epoch": 0.3994874500640687,
748
+ "grad_norm": 0.19991905987262726,
749
+ "learning_rate": 9.047449824095075e-05,
750
+ "loss": 0.4939,
751
+ "step": 5300
752
+ },
753
+ {
754
+ "epoch": 0.40325619959297504,
755
+ "grad_norm": 0.1946549117565155,
756
+ "learning_rate": 9.030000027380012e-05,
757
+ "loss": 0.4978,
758
+ "step": 5350
759
+ },
760
+ {
761
+ "epoch": 0.40702494912188136,
762
+ "grad_norm": 0.18962378799915314,
763
+ "learning_rate": 9.012408997046766e-05,
764
+ "loss": 0.4978,
765
+ "step": 5400
766
+ },
767
+ {
768
+ "epoch": 0.4107936986507877,
769
+ "grad_norm": 0.1853850781917572,
770
+ "learning_rate": 8.994677349582888e-05,
771
+ "loss": 0.4971,
772
+ "step": 5450
773
+ },
774
+ {
775
+ "epoch": 0.41456244817969395,
776
+ "grad_norm": 0.19636128842830658,
777
+ "learning_rate": 8.976805706403942e-05,
778
+ "loss": 0.4979,
779
+ "step": 5500
780
+ },
781
+ {
782
+ "epoch": 0.4183311977086003,
783
+ "grad_norm": 0.19317063689231873,
784
+ "learning_rate": 8.958794693831716e-05,
785
+ "loss": 0.4963,
786
+ "step": 5550
787
+ },
788
+ {
789
+ "epoch": 0.4220999472375066,
790
+ "grad_norm": 0.1992931365966797,
791
+ "learning_rate": 8.94064494307228e-05,
792
+ "loss": 0.4947,
793
+ "step": 5600
794
+ },
795
+ {
796
+ "epoch": 0.4258686967664129,
797
+ "grad_norm": 0.19183537364006042,
798
+ "learning_rate": 8.922357090193858e-05,
799
+ "loss": 0.4964,
800
+ "step": 5650
801
+ },
802
+ {
803
+ "epoch": 0.4296374462953192,
804
+ "grad_norm": 0.20117546617984772,
805
+ "learning_rate": 8.903931776104545e-05,
806
+ "loss": 0.4949,
807
+ "step": 5700
808
+ },
809
+ {
810
+ "epoch": 0.4334061958242255,
811
+ "grad_norm": 0.19739404320716858,
812
+ "learning_rate": 8.885369646529838e-05,
813
+ "loss": 0.4965,
814
+ "step": 5750
815
+ },
816
+ {
817
+ "epoch": 0.43717494535313184,
818
+ "grad_norm": 0.2107592076063156,
819
+ "learning_rate": 8.866671351990007e-05,
820
+ "loss": 0.4957,
821
+ "step": 5800
822
+ },
823
+ {
824
+ "epoch": 0.44094369488203816,
825
+ "grad_norm": 0.20069849491119385,
826
+ "learning_rate": 8.847837547777312e-05,
827
+ "loss": 0.4967,
828
+ "step": 5850
829
+ },
830
+ {
831
+ "epoch": 0.44471244441094443,
832
+ "grad_norm": 0.19208498299121857,
833
+ "learning_rate": 8.82886889393301e-05,
834
+ "loss": 0.4934,
835
+ "step": 5900
836
+ },
837
+ {
838
+ "epoch": 0.44848119393985075,
839
+ "grad_norm": 0.1858828216791153,
840
+ "learning_rate": 8.80976605522425e-05,
841
+ "loss": 0.4963,
842
+ "step": 5950
843
+ },
844
+ {
845
+ "epoch": 0.4522499434687571,
846
+ "grad_norm": 0.1936417669057846,
847
+ "learning_rate": 8.790529701120759e-05,
848
+ "loss": 0.4943,
849
+ "step": 6000
850
+ },
851
+ {
852
+ "epoch": 0.4560186929976634,
853
+ "grad_norm": 0.20013850927352905,
854
+ "learning_rate": 8.771160505771391e-05,
855
+ "loss": 0.4928,
856
+ "step": 6050
857
+ },
858
+ {
859
+ "epoch": 0.45978744252656967,
860
+ "grad_norm": 0.1874343603849411,
861
+ "learning_rate": 8.751659147980493e-05,
862
+ "loss": 0.4929,
863
+ "step": 6100
864
+ },
865
+ {
866
+ "epoch": 0.463556192055476,
867
+ "grad_norm": 0.1926513910293579,
868
+ "learning_rate": 8.732026311184119e-05,
869
+ "loss": 0.4928,
870
+ "step": 6150
871
+ },
872
+ {
873
+ "epoch": 0.4673249415843823,
874
+ "grad_norm": 0.19659832119941711,
875
+ "learning_rate": 8.712262683426082e-05,
876
+ "loss": 0.4924,
877
+ "step": 6200
878
+ },
879
+ {
880
+ "epoch": 0.47109369111328864,
881
+ "grad_norm": 0.1873038113117218,
882
+ "learning_rate": 8.692368957333834e-05,
883
+ "loss": 0.4888,
884
+ "step": 6250
885
+ },
886
+ {
887
+ "epoch": 0.4748624406421949,
888
+ "grad_norm": 0.1897285133600235,
889
+ "learning_rate": 8.672345830094199e-05,
890
+ "loss": 0.4946,
891
+ "step": 6300
892
+ },
893
+ {
894
+ "epoch": 0.47863119017110123,
895
+ "grad_norm": 0.18970079720020294,
896
+ "learning_rate": 8.652194003428932e-05,
897
+ "loss": 0.4909,
898
+ "step": 6350
899
+ },
900
+ {
901
+ "epoch": 0.48239993970000755,
902
+ "grad_norm": 0.18571080267429352,
903
+ "learning_rate": 8.631914183570143e-05,
904
+ "loss": 0.4917,
905
+ "step": 6400
906
+ },
907
+ {
908
+ "epoch": 0.4861686892289138,
909
+ "grad_norm": 0.19028496742248535,
910
+ "learning_rate": 8.611507081235522e-05,
911
+ "loss": 0.4927,
912
+ "step": 6450
913
+ },
914
+ {
915
+ "epoch": 0.48993743875782014,
916
+ "grad_norm": 0.1985916942358017,
917
+ "learning_rate": 8.590973411603452e-05,
918
+ "loss": 0.4921,
919
+ "step": 6500
920
+ },
921
+ {
922
+ "epoch": 0.49370618828672647,
923
+ "grad_norm": 0.19005028903484344,
924
+ "learning_rate": 8.570313894287938e-05,
925
+ "loss": 0.4897,
926
+ "step": 6550
927
+ },
928
+ {
929
+ "epoch": 0.4974749378156328,
930
+ "grad_norm": 0.18022720515727997,
931
+ "learning_rate": 8.549529253313386e-05,
932
+ "loss": 0.4903,
933
+ "step": 6600
934
+ },
935
+ {
936
+ "epoch": 0.5012436873445391,
937
+ "grad_norm": 0.20154112577438354,
938
+ "learning_rate": 8.528620217089236e-05,
939
+ "loss": 0.4907,
940
+ "step": 6650
941
+ },
942
+ {
943
+ "epoch": 0.5050124368734454,
944
+ "grad_norm": 0.1906825751066208,
945
+ "learning_rate": 8.507587518384421e-05,
946
+ "loss": 0.4895,
947
+ "step": 6700
948
+ },
949
+ {
950
+ "epoch": 0.5087811864023517,
951
+ "grad_norm": 0.2176685482263565,
952
+ "learning_rate": 8.486431894301704e-05,
953
+ "loss": 0.4902,
954
+ "step": 6750
955
+ },
956
+ {
957
+ "epoch": 0.512549935931258,
958
+ "grad_norm": 0.19985392689704895,
959
+ "learning_rate": 8.465154086251828e-05,
960
+ "loss": 0.4892,
961
+ "step": 6800
962
+ },
963
+ {
964
+ "epoch": 0.5163186854601644,
965
+ "grad_norm": 0.2009229063987732,
966
+ "learning_rate": 8.443754839927549e-05,
967
+ "loss": 0.4876,
968
+ "step": 6850
969
+ },
970
+ {
971
+ "epoch": 0.5200874349890706,
972
+ "grad_norm": 0.18816043436527252,
973
+ "learning_rate": 8.422234905277495e-05,
974
+ "loss": 0.4878,
975
+ "step": 6900
976
+ },
977
+ {
978
+ "epoch": 0.5238561845179769,
979
+ "grad_norm": 0.1980554759502411,
980
+ "learning_rate": 8.400595036479881e-05,
981
+ "loss": 0.4876,
982
+ "step": 6950
983
+ },
984
+ {
985
+ "epoch": 0.5276249340468833,
986
+ "grad_norm": 0.19587980210781097,
987
+ "learning_rate": 8.378835991916083e-05,
988
+ "loss": 0.4883,
989
+ "step": 7000
990
+ },
991
+ {
992
+ "epoch": 0.5313936835757895,
993
+ "grad_norm": 0.19175052642822266,
994
+ "learning_rate": 8.35695853414406e-05,
995
+ "loss": 0.4879,
996
+ "step": 7050
997
+ },
998
+ {
999
+ "epoch": 0.5351624331046959,
1000
+ "grad_norm": 0.1843547224998474,
1001
+ "learning_rate": 8.334963429871627e-05,
1002
+ "loss": 0.4897,
1003
+ "step": 7100
1004
+ },
1005
+ {
1006
+ "epoch": 0.5389311826336022,
1007
+ "grad_norm": 0.18825556337833405,
1008
+ "learning_rate": 8.312851449929589e-05,
1009
+ "loss": 0.4884,
1010
+ "step": 7150
1011
+ },
1012
+ {
1013
+ "epoch": 0.5426999321625084,
1014
+ "grad_norm": 0.1980101317167282,
1015
+ "learning_rate": 8.290623369244721e-05,
1016
+ "loss": 0.4844,
1017
+ "step": 7200
1018
+ },
1019
+ {
1020
+ "epoch": 0.5464686816914148,
1021
+ "grad_norm": 0.2058933824300766,
1022
+ "learning_rate": 8.268279966812617e-05,
1023
+ "loss": 0.4854,
1024
+ "step": 7250
1025
+ },
1026
+ {
1027
+ "epoch": 0.5502374312203211,
1028
+ "grad_norm": 0.17871695756912231,
1029
+ "learning_rate": 8.245822025670384e-05,
1030
+ "loss": 0.4855,
1031
+ "step": 7300
1032
+ },
1033
+ {
1034
+ "epoch": 0.5540061807492274,
1035
+ "grad_norm": 0.18174917995929718,
1036
+ "learning_rate": 8.223250332869206e-05,
1037
+ "loss": 0.4861,
1038
+ "step": 7350
1039
+ },
1040
+ {
1041
+ "epoch": 0.5577749302781337,
1042
+ "grad_norm": 0.19006958603858948,
1043
+ "learning_rate": 8.200565679446753e-05,
1044
+ "loss": 0.4872,
1045
+ "step": 7400
1046
+ },
1047
+ {
1048
+ "epoch": 0.56154367980704,
1049
+ "grad_norm": 0.188937246799469,
1050
+ "learning_rate": 8.177768860399467e-05,
1051
+ "loss": 0.4832,
1052
+ "step": 7450
1053
+ },
1054
+ {
1055
+ "epoch": 0.5653124293359464,
1056
+ "grad_norm": 0.19133251905441284,
1057
+ "learning_rate": 8.154860674654698e-05,
1058
+ "loss": 0.4867,
1059
+ "step": 7500
1060
+ },
1061
+ {
1062
+ "epoch": 0.5690811788648527,
1063
+ "grad_norm": 0.18221163749694824,
1064
+ "learning_rate": 8.131841925042704e-05,
1065
+ "loss": 0.4846,
1066
+ "step": 7550
1067
+ },
1068
+ {
1069
+ "epoch": 0.5728499283937589,
1070
+ "grad_norm": 0.18946631252765656,
1071
+ "learning_rate": 8.108713418268514e-05,
1072
+ "loss": 0.4847,
1073
+ "step": 7600
1074
+ },
1075
+ {
1076
+ "epoch": 0.5766186779226653,
1077
+ "grad_norm": 0.18349218368530273,
1078
+ "learning_rate": 8.085475964883657e-05,
1079
+ "loss": 0.4839,
1080
+ "step": 7650
1081
+ },
1082
+ {
1083
+ "epoch": 0.5803874274515716,
1084
+ "grad_norm": 0.19051064550876617,
1085
+ "learning_rate": 8.062130379257764e-05,
1086
+ "loss": 0.485,
1087
+ "step": 7700
1088
+ },
1089
+ {
1090
+ "epoch": 0.5841561769804778,
1091
+ "grad_norm": 0.18272393941879272,
1092
+ "learning_rate": 8.038677479550016e-05,
1093
+ "loss": 0.4844,
1094
+ "step": 7750
1095
+ },
1096
+ {
1097
+ "epoch": 0.5879249265093842,
1098
+ "grad_norm": 0.17754477262496948,
1099
+ "learning_rate": 8.015118087680477e-05,
1100
+ "loss": 0.4838,
1101
+ "step": 7800
1102
+ },
1103
+ {
1104
+ "epoch": 0.5916936760382905,
1105
+ "grad_norm": 0.18516160547733307,
1106
+ "learning_rate": 7.99145302930129e-05,
1107
+ "loss": 0.4823,
1108
+ "step": 7850
1109
+ },
1110
+ {
1111
+ "epoch": 0.5954624255671968,
1112
+ "grad_norm": 0.18762782216072083,
1113
+ "learning_rate": 7.96768313376774e-05,
1114
+ "loss": 0.4819,
1115
+ "step": 7900
1116
+ },
1117
+ {
1118
+ "epoch": 0.5992311750961031,
1119
+ "grad_norm": 0.18815983831882477,
1120
+ "learning_rate": 7.943809234109185e-05,
1121
+ "loss": 0.4813,
1122
+ "step": 7950
1123
+ },
1124
+ {
1125
+ "epoch": 0.6029999246250094,
1126
+ "grad_norm": 0.19032306969165802,
1127
+ "learning_rate": 7.919832166999874e-05,
1128
+ "loss": 0.4798,
1129
+ "step": 8000
1130
+ },
1131
+ {
1132
+ "epoch": 0.6067686741539158,
1133
+ "grad_norm": 0.18525487184524536,
1134
+ "learning_rate": 7.895752772729612e-05,
1135
+ "loss": 0.4804,
1136
+ "step": 8050
1137
+ },
1138
+ {
1139
+ "epoch": 0.610537423682822,
1140
+ "grad_norm": 0.18313206732273102,
1141
+ "learning_rate": 7.871571895174316e-05,
1142
+ "loss": 0.4804,
1143
+ "step": 8100
1144
+ },
1145
+ {
1146
+ "epoch": 0.6143061732117283,
1147
+ "grad_norm": 0.19534072279930115,
1148
+ "learning_rate": 7.847290381766446e-05,
1149
+ "loss": 0.4807,
1150
+ "step": 8150
1151
+ },
1152
+ {
1153
+ "epoch": 0.6180749227406347,
1154
+ "grad_norm": 0.1794072687625885,
1155
+ "learning_rate": 7.822909083465298e-05,
1156
+ "loss": 0.481,
1157
+ "step": 8200
1158
+ },
1159
+ {
1160
+ "epoch": 0.621843672269541,
1161
+ "grad_norm": 0.18725860118865967,
1162
+ "learning_rate": 7.798428854727188e-05,
1163
+ "loss": 0.4803,
1164
+ "step": 8250
1165
+ },
1166
+ {
1167
+ "epoch": 0.6256124217984472,
1168
+ "grad_norm": 0.190611332654953,
1169
+ "learning_rate": 7.773850553475508e-05,
1170
+ "loss": 0.4802,
1171
+ "step": 8300
1172
+ },
1173
+ {
1174
+ "epoch": 0.6293811713273536,
1175
+ "grad_norm": 0.18728849291801453,
1176
+ "learning_rate": 7.749175041070647e-05,
1177
+ "loss": 0.4787,
1178
+ "step": 8350
1179
+ },
1180
+ {
1181
+ "epoch": 0.6331499208562599,
1182
+ "grad_norm": 0.19463524222373962,
1183
+ "learning_rate": 7.724403182279823e-05,
1184
+ "loss": 0.4791,
1185
+ "step": 8400
1186
+ },
1187
+ {
1188
+ "epoch": 0.6369186703851663,
1189
+ "grad_norm": 0.1864539533853531,
1190
+ "learning_rate": 7.699535845246761e-05,
1191
+ "loss": 0.4787,
1192
+ "step": 8450
1193
+ },
1194
+ {
1195
+ "epoch": 0.6406874199140725,
1196
+ "grad_norm": 0.18798550963401794,
1197
+ "learning_rate": 7.674573901461282e-05,
1198
+ "loss": 0.4773,
1199
+ "step": 8500
1200
+ },
1201
+ {
1202
+ "epoch": 0.6444561694429788,
1203
+ "grad_norm": 0.18946604430675507,
1204
+ "learning_rate": 7.649518225728745e-05,
1205
+ "loss": 0.4803,
1206
+ "step": 8550
1207
+ },
1208
+ {
1209
+ "epoch": 0.6482249189718852,
1210
+ "grad_norm": 0.19057536125183105,
1211
+ "learning_rate": 7.624369696139402e-05,
1212
+ "loss": 0.4785,
1213
+ "step": 8600
1214
+ },
1215
+ {
1216
+ "epoch": 0.6519936685007914,
1217
+ "grad_norm": 0.18735001981258392,
1218
+ "learning_rate": 7.599129194037624e-05,
1219
+ "loss": 0.4795,
1220
+ "step": 8650
1221
+ },
1222
+ {
1223
+ "epoch": 0.6557624180296977,
1224
+ "grad_norm": 0.19279134273529053,
1225
+ "learning_rate": 7.573797603991004e-05,
1226
+ "loss": 0.4775,
1227
+ "step": 8700
1228
+ },
1229
+ {
1230
+ "epoch": 0.6595311675586041,
1231
+ "grad_norm": 0.18735423684120178,
1232
+ "learning_rate": 7.548375813759366e-05,
1233
+ "loss": 0.478,
1234
+ "step": 8750
1235
+ },
1236
+ {
1237
+ "epoch": 0.6632999170875103,
1238
+ "grad_norm": 0.1862768828868866,
1239
+ "learning_rate": 7.522864714263655e-05,
1240
+ "loss": 0.478,
1241
+ "step": 8800
1242
+ },
1243
+ {
1244
+ "epoch": 0.6670686666164167,
1245
+ "grad_norm": 0.18440242111682892,
1246
+ "learning_rate": 7.497265199554699e-05,
1247
+ "loss": 0.4776,
1248
+ "step": 8850
1249
+ },
1250
+ {
1251
+ "epoch": 0.670837416145323,
1252
+ "grad_norm": 0.181020587682724,
1253
+ "learning_rate": 7.471578166781899e-05,
1254
+ "loss": 0.4738,
1255
+ "step": 8900
1256
+ },
1257
+ {
1258
+ "epoch": 0.6746061656742293,
1259
+ "grad_norm": 0.18536238372325897,
1260
+ "learning_rate": 7.445804516161768e-05,
1261
+ "loss": 0.4761,
1262
+ "step": 8950
1263
+ },
1264
+ {
1265
+ "epoch": 0.6783749152031356,
1266
+ "grad_norm": 0.1886385977268219,
1267
+ "learning_rate": 7.419945150946386e-05,
1268
+ "loss": 0.4773,
1269
+ "step": 9000
1270
+ },
1271
+ {
1272
+ "epoch": 0.6821436647320419,
1273
+ "grad_norm": 0.1746935397386551,
1274
+ "learning_rate": 7.394000977391764e-05,
1275
+ "loss": 0.4764,
1276
+ "step": 9050
1277
+ },
1278
+ {
1279
+ "epoch": 0.6859124142609482,
1280
+ "grad_norm": 0.18953968584537506,
1281
+ "learning_rate": 7.367972904726055e-05,
1282
+ "loss": 0.4766,
1283
+ "step": 9100
1284
+ },
1285
+ {
1286
+ "epoch": 0.6896811637898546,
1287
+ "grad_norm": 0.47858861088752747,
1288
+ "learning_rate": 7.341861845117708e-05,
1289
+ "loss": 0.4742,
1290
+ "step": 9150
1291
+ },
1292
+ {
1293
+ "epoch": 0.6934499133187608,
1294
+ "grad_norm": 0.19248232245445251,
1295
+ "learning_rate": 7.3156687136435e-05,
1296
+ "loss": 0.4753,
1297
+ "step": 9200
1298
+ },
1299
+ {
1300
+ "epoch": 0.6972186628476671,
1301
+ "grad_norm": 0.18838423490524292,
1302
+ "learning_rate": 7.289394428256456e-05,
1303
+ "loss": 0.4747,
1304
+ "step": 9250
1305
+ },
1306
+ {
1307
+ "epoch": 0.7009874123765735,
1308
+ "grad_norm": 0.18120819330215454,
1309
+ "learning_rate": 7.26303990975369e-05,
1310
+ "loss": 0.476,
1311
+ "step": 9300
1312
+ },
1313
+ {
1314
+ "epoch": 0.7047561619054797,
1315
+ "grad_norm": 0.18532796204090118,
1316
+ "learning_rate": 7.236606081744132e-05,
1317
+ "loss": 0.4766,
1318
+ "step": 9350
1319
+ },
1320
+ {
1321
+ "epoch": 0.7085249114343861,
1322
+ "grad_norm": 0.19351600110530853,
1323
+ "learning_rate": 7.210093870616155e-05,
1324
+ "loss": 0.4771,
1325
+ "step": 9400
1326
+ },
1327
+ {
1328
+ "epoch": 0.7122936609632924,
1329
+ "grad_norm": 0.18240147829055786,
1330
+ "learning_rate": 7.183504205505111e-05,
1331
+ "loss": 0.4747,
1332
+ "step": 9450
1333
+ },
1334
+ {
1335
+ "epoch": 0.7160624104921987,
1336
+ "grad_norm": 0.18519514799118042,
1337
+ "learning_rate": 7.156838018260776e-05,
1338
+ "loss": 0.4731,
1339
+ "step": 9500
1340
+ },
1341
+ {
1342
+ "epoch": 0.719831160021105,
1343
+ "grad_norm": 0.192172572016716,
1344
+ "learning_rate": 7.13009624341468e-05,
1345
+ "loss": 0.4724,
1346
+ "step": 9550
1347
+ },
1348
+ {
1349
+ "epoch": 0.7235999095500113,
1350
+ "grad_norm": 0.18007811903953552,
1351
+ "learning_rate": 7.103279818147371e-05,
1352
+ "loss": 0.4725,
1353
+ "step": 9600
1354
+ },
1355
+ {
1356
+ "epoch": 0.7273686590789176,
1357
+ "grad_norm": 0.18547901511192322,
1358
+ "learning_rate": 7.076389682255556e-05,
1359
+ "loss": 0.4739,
1360
+ "step": 9650
1361
+ },
1362
+ {
1363
+ "epoch": 0.731137408607824,
1364
+ "grad_norm": 0.18853244185447693,
1365
+ "learning_rate": 7.049426778119179e-05,
1366
+ "loss": 0.4748,
1367
+ "step": 9700
1368
+ },
1369
+ {
1370
+ "epoch": 0.7349061581367302,
1371
+ "grad_norm": 0.17960478365421295,
1372
+ "learning_rate": 7.022392050668381e-05,
1373
+ "loss": 0.4728,
1374
+ "step": 9750
1375
+ },
1376
+ {
1377
+ "epoch": 0.7386749076656366,
1378
+ "grad_norm": 0.19630731642246246,
1379
+ "learning_rate": 6.995286447350397e-05,
1380
+ "loss": 0.4713,
1381
+ "step": 9800
1382
+ },
1383
+ {
1384
+ "epoch": 0.7424436571945429,
1385
+ "grad_norm": 0.1827552616596222,
1386
+ "learning_rate": 6.968110918096342e-05,
1387
+ "loss": 0.4707,
1388
+ "step": 9850
1389
+ },
1390
+ {
1391
+ "epoch": 0.7462124067234491,
1392
+ "grad_norm": 0.18695256114006042,
1393
+ "learning_rate": 6.940866415287931e-05,
1394
+ "loss": 0.4704,
1395
+ "step": 9900
1396
+ },
1397
+ {
1398
+ "epoch": 0.7499811562523555,
1399
+ "grad_norm": 0.19157296419143677,
1400
+ "learning_rate": 6.91355389372409e-05,
1401
+ "loss": 0.4728,
1402
+ "step": 9950
1403
+ },
1404
+ {
1405
+ "epoch": 0.7537499057812618,
1406
+ "grad_norm": 0.18015547096729279,
1407
+ "learning_rate": 6.886174310587501e-05,
1408
+ "loss": 0.4732,
1409
+ "step": 10000
1410
+ },
1411
+ {
1412
+ "epoch": 0.757518655310168,
1413
+ "grad_norm": 0.19139304757118225,
1414
+ "learning_rate": 6.858728625411064e-05,
1415
+ "loss": 0.4713,
1416
+ "step": 10050
1417
+ },
1418
+ {
1419
+ "epoch": 0.7612874048390744,
1420
+ "grad_norm": 0.18222898244857788,
1421
+ "learning_rate": 6.831217800044252e-05,
1422
+ "loss": 0.4707,
1423
+ "step": 10100
1424
+ },
1425
+ {
1426
+ "epoch": 0.7650561543679807,
1427
+ "grad_norm": 0.196458101272583,
1428
+ "learning_rate": 6.803642798619418e-05,
1429
+ "loss": 0.4687,
1430
+ "step": 10150
1431
+ },
1432
+ {
1433
+ "epoch": 0.768824903896887,
1434
+ "grad_norm": 0.2024402916431427,
1435
+ "learning_rate": 6.776004587518001e-05,
1436
+ "loss": 0.4707,
1437
+ "step": 10200
1438
+ },
1439
+ {
1440
+ "epoch": 0.7725936534257933,
1441
+ "grad_norm": 0.18663807213306427,
1442
+ "learning_rate": 6.748304135336658e-05,
1443
+ "loss": 0.468,
1444
+ "step": 10250
1445
+ },
1446
+ {
1447
+ "epoch": 0.7763624029546996,
1448
+ "grad_norm": 0.18797928094863892,
1449
+ "learning_rate": 6.720542412853319e-05,
1450
+ "loss": 0.4701,
1451
+ "step": 10300
1452
+ },
1453
+ {
1454
+ "epoch": 0.780131152483606,
1455
+ "grad_norm": 0.18169479072093964,
1456
+ "learning_rate": 6.692720392993166e-05,
1457
+ "loss": 0.4666,
1458
+ "step": 10350
1459
+ },
1460
+ {
1461
+ "epoch": 0.7838999020125123,
1462
+ "grad_norm": 0.17825226485729218,
1463
+ "learning_rate": 6.66483905079454e-05,
1464
+ "loss": 0.4687,
1465
+ "step": 10400
1466
+ },
1467
+ {
1468
+ "epoch": 0.7876686515414185,
1469
+ "grad_norm": 0.18662355840206146,
1470
+ "learning_rate": 6.636899363374763e-05,
1471
+ "loss": 0.4687,
1472
+ "step": 10450
1473
+ },
1474
+ {
1475
+ "epoch": 0.7914374010703249,
1476
+ "grad_norm": 0.19263768196105957,
1477
+ "learning_rate": 6.608902309895895e-05,
1478
+ "loss": 0.4675,
1479
+ "step": 10500
1480
+ },
1481
+ {
1482
+ "epoch": 0.7952061505992312,
1483
+ "grad_norm": 0.1900280863046646,
1484
+ "learning_rate": 6.580848871530433e-05,
1485
+ "loss": 0.4694,
1486
+ "step": 10550
1487
+ },
1488
+ {
1489
+ "epoch": 0.7989749001281374,
1490
+ "grad_norm": 0.17986111342906952,
1491
+ "learning_rate": 6.552740031426902e-05,
1492
+ "loss": 0.4658,
1493
+ "step": 10600
1494
+ },
1495
+ {
1496
+ "epoch": 0.8027436496570438,
1497
+ "grad_norm": 0.1776350736618042,
1498
+ "learning_rate": 6.52457677467542e-05,
1499
+ "loss": 0.4696,
1500
+ "step": 10650
1501
+ },
1502
+ {
1503
+ "epoch": 0.8065123991859501,
1504
+ "grad_norm": 0.19290950894355774,
1505
+ "learning_rate": 6.496360088273161e-05,
1506
+ "loss": 0.4675,
1507
+ "step": 10700
1508
+ },
1509
+ {
1510
+ "epoch": 0.8102811487148565,
1511
+ "grad_norm": 0.17263682186603546,
1512
+ "learning_rate": 6.468090961089775e-05,
1513
+ "loss": 0.4672,
1514
+ "step": 10750
1515
+ },
1516
+ {
1517
+ "epoch": 0.8140498982437627,
1518
+ "grad_norm": 0.18622560799121857,
1519
+ "learning_rate": 6.439770383832732e-05,
1520
+ "loss": 0.4664,
1521
+ "step": 10800
1522
+ },
1523
+ {
1524
+ "epoch": 0.817818647772669,
1525
+ "grad_norm": 0.18446871638298035,
1526
+ "learning_rate": 6.41139934901259e-05,
1527
+ "loss": 0.4668,
1528
+ "step": 10850
1529
+ },
1530
+ {
1531
+ "epoch": 0.8215873973015754,
1532
+ "grad_norm": 0.1856282651424408,
1533
+ "learning_rate": 6.382978850908226e-05,
1534
+ "loss": 0.466,
1535
+ "step": 10900
1536
+ },
1537
+ {
1538
+ "epoch": 0.8253561468304816,
1539
+ "grad_norm": 0.17993173003196716,
1540
+ "learning_rate": 6.354509885531989e-05,
1541
+ "loss": 0.4653,
1542
+ "step": 10950
1543
+ },
1544
+ {
1545
+ "epoch": 0.8291248963593879,
1546
+ "grad_norm": 0.1821722388267517,
1547
+ "learning_rate": 6.325993450594782e-05,
1548
+ "loss": 0.4653,
1549
+ "step": 11000
1550
+ },
1551
+ {
1552
+ "epoch": 0.8328936458882943,
1553
+ "grad_norm": 0.18031588196754456,
1554
+ "learning_rate": 6.297430545471112e-05,
1555
+ "loss": 0.4651,
1556
+ "step": 11050
1557
+ },
1558
+ {
1559
+ "epoch": 0.8366623954172006,
1560
+ "grad_norm": 0.18513865768909454,
1561
+ "learning_rate": 6.26882217116406e-05,
1562
+ "loss": 0.4664,
1563
+ "step": 11100
1564
+ },
1565
+ {
1566
+ "epoch": 0.8404311449461069,
1567
+ "grad_norm": 0.18106934428215027,
1568
+ "learning_rate": 6.240169330270197e-05,
1569
+ "loss": 0.4636,
1570
+ "step": 11150
1571
+ },
1572
+ {
1573
+ "epoch": 0.8441998944750132,
1574
+ "grad_norm": 0.1788669228553772,
1575
+ "learning_rate": 6.211473026944452e-05,
1576
+ "loss": 0.4655,
1577
+ "step": 11200
1578
+ },
1579
+ {
1580
+ "epoch": 0.8479686440039195,
1581
+ "grad_norm": 0.19618336856365204,
1582
+ "learning_rate": 6.18273426686492e-05,
1583
+ "loss": 0.4663,
1584
+ "step": 11250
1585
+ },
1586
+ {
1587
+ "epoch": 0.8517373935328258,
1588
+ "grad_norm": 0.1805189996957779,
1589
+ "learning_rate": 6.153954057197612e-05,
1590
+ "loss": 0.4638,
1591
+ "step": 11300
1592
+ },
1593
+ {
1594
+ "epoch": 0.8555061430617321,
1595
+ "grad_norm": 0.17371240258216858,
1596
+ "learning_rate": 6.125133406561172e-05,
1597
+ "loss": 0.4631,
1598
+ "step": 11350
1599
+ },
1600
+ {
1601
+ "epoch": 0.8592748925906384,
1602
+ "grad_norm": 0.18736299872398376,
1603
+ "learning_rate": 6.0962733249915135e-05,
1604
+ "loss": 0.4626,
1605
+ "step": 11400
1606
+ },
1607
+ {
1608
+ "epoch": 0.8630436421195448,
1609
+ "grad_norm": 0.19724301993846893,
1610
+ "learning_rate": 6.067374823906431e-05,
1611
+ "loss": 0.462,
1612
+ "step": 11450
1613
+ },
1614
+ {
1615
+ "epoch": 0.866812391648451,
1616
+ "grad_norm": 0.18249565362930298,
1617
+ "learning_rate": 6.038438916070155e-05,
1618
+ "loss": 0.463,
1619
+ "step": 11500
1620
+ },
1621
+ {
1622
+ "epoch": 0.8705811411773573,
1623
+ "grad_norm": 0.17936116456985474,
1624
+ "learning_rate": 6.0094666155578525e-05,
1625
+ "loss": 0.4634,
1626
+ "step": 11550
1627
+ },
1628
+ {
1629
+ "epoch": 0.8743498907062637,
1630
+ "grad_norm": 0.18539419770240784,
1631
+ "learning_rate": 5.9804589377200946e-05,
1632
+ "loss": 0.4614,
1633
+ "step": 11600
1634
+ },
1635
+ {
1636
+ "epoch": 0.87811864023517,
1637
+ "grad_norm": 0.18874451518058777,
1638
+ "learning_rate": 5.951416899147273e-05,
1639
+ "loss": 0.4615,
1640
+ "step": 11650
1641
+ },
1642
+ {
1643
+ "epoch": 0.8818873897640763,
1644
+ "grad_norm": 0.19202075898647308,
1645
+ "learning_rate": 5.922341517633965e-05,
1646
+ "loss": 0.4647,
1647
+ "step": 11700
1648
+ },
1649
+ {
1650
+ "epoch": 0.8856561392929826,
1651
+ "grad_norm": 0.17386788129806519,
1652
+ "learning_rate": 5.893233812143274e-05,
1653
+ "loss": 0.4608,
1654
+ "step": 11750
1655
+ },
1656
+ {
1657
+ "epoch": 0.8894248888218889,
1658
+ "grad_norm": 0.1823577582836151,
1659
+ "learning_rate": 5.864094802771115e-05,
1660
+ "loss": 0.4609,
1661
+ "step": 11800
1662
+ },
1663
+ {
1664
+ "epoch": 0.8931936383507952,
1665
+ "grad_norm": 0.1865212470293045,
1666
+ "learning_rate": 5.834925510710465e-05,
1667
+ "loss": 0.4607,
1668
+ "step": 11850
1669
+ },
1670
+ {
1671
+ "epoch": 0.8969623878797015,
1672
+ "grad_norm": 0.18660889565944672,
1673
+ "learning_rate": 5.8057269582155735e-05,
1674
+ "loss": 0.4629,
1675
+ "step": 11900
1676
+ },
1677
+ {
1678
+ "epoch": 0.9007311374086078,
1679
+ "grad_norm": 0.17511461675167084,
1680
+ "learning_rate": 5.776500168566141e-05,
1681
+ "loss": 0.4591,
1682
+ "step": 11950
1683
+ },
1684
+ {
1685
+ "epoch": 0.9044998869375142,
1686
+ "grad_norm": 0.17838037014007568,
1687
+ "learning_rate": 5.7472461660314504e-05,
1688
+ "loss": 0.4609,
1689
+ "step": 12000
1690
+ },
1691
+ {
1692
+ "epoch": 0.9082686364664204,
1693
+ "grad_norm": 0.18015101552009583,
1694
+ "learning_rate": 5.717965975834483e-05,
1695
+ "loss": 0.4607,
1696
+ "step": 12050
1697
+ },
1698
+ {
1699
+ "epoch": 0.9120373859953268,
1700
+ "grad_norm": 0.169195756316185,
1701
+ "learning_rate": 5.6886606241159714e-05,
1702
+ "loss": 0.4582,
1703
+ "step": 12100
1704
+ },
1705
+ {
1706
+ "epoch": 0.9158061355242331,
1707
+ "grad_norm": 0.1795186996459961,
1708
+ "learning_rate": 5.659331137898458e-05,
1709
+ "loss": 0.461,
1710
+ "step": 12150
1711
+ },
1712
+ {
1713
+ "epoch": 0.9195748850531393,
1714
+ "grad_norm": 0.1830703467130661,
1715
+ "learning_rate": 5.6299785450502853e-05,
1716
+ "loss": 0.4594,
1717
+ "step": 12200
1718
+ },
1719
+ {
1720
+ "epoch": 0.9233436345820457,
1721
+ "grad_norm": 0.1865684539079666,
1722
+ "learning_rate": 5.600603874249585e-05,
1723
+ "loss": 0.4602,
1724
+ "step": 12250
1725
+ },
1726
+ {
1727
+ "epoch": 0.927112384110952,
1728
+ "grad_norm": 0.17610585689544678,
1729
+ "learning_rate": 5.571208154948218e-05,
1730
+ "loss": 0.4611,
1731
+ "step": 12300
1732
+ },
1733
+ {
1734
+ "epoch": 0.9308811336398582,
1735
+ "grad_norm": 0.18364217877388,
1736
+ "learning_rate": 5.541792417335707e-05,
1737
+ "loss": 0.4599,
1738
+ "step": 12350
1739
+ },
1740
+ {
1741
+ "epoch": 0.9346498831687646,
1742
+ "grad_norm": 0.1822074055671692,
1743
+ "learning_rate": 5.5123576923031253e-05,
1744
+ "loss": 0.4599,
1745
+ "step": 12400
1746
+ },
1747
+ {
1748
+ "epoch": 0.9384186326976709,
1749
+ "grad_norm": 0.18282079696655273,
1750
+ "learning_rate": 5.482905011406973e-05,
1751
+ "loss": 0.4568,
1752
+ "step": 12450
1753
+ },
1754
+ {
1755
+ "epoch": 0.9421873822265773,
1756
+ "grad_norm": 0.18633867800235748,
1757
+ "learning_rate": 5.453435406833017e-05,
1758
+ "loss": 0.457,
1759
+ "step": 12500
1760
+ },
1761
+ {
1762
+ "epoch": 0.9459561317554835,
1763
+ "grad_norm": 0.17714855074882507,
1764
+ "learning_rate": 5.4239499113601333e-05,
1765
+ "loss": 0.4558,
1766
+ "step": 12550
1767
+ },
1768
+ {
1769
+ "epoch": 0.9497248812843898,
1770
+ "grad_norm": 0.1829010248184204,
1771
+ "learning_rate": 5.3944495583240987e-05,
1772
+ "loss": 0.4591,
1773
+ "step": 12600
1774
+ },
1775
+ {
1776
+ "epoch": 0.9534936308132962,
1777
+ "grad_norm": 0.18278288841247559,
1778
+ "learning_rate": 5.364935381581377e-05,
1779
+ "loss": 0.4592,
1780
+ "step": 12650
1781
+ },
1782
+ {
1783
+ "epoch": 0.9572623803422025,
1784
+ "grad_norm": 0.18331314623355865,
1785
+ "learning_rate": 5.3354084154729034e-05,
1786
+ "loss": 0.4573,
1787
+ "step": 12700
1788
+ },
1789
+ {
1790
+ "epoch": 0.9610311298711087,
1791
+ "grad_norm": 0.19474105536937714,
1792
+ "learning_rate": 5.305869694787812e-05,
1793
+ "loss": 0.4557,
1794
+ "step": 12750
1795
+ },
1796
+ {
1797
+ "epoch": 0.9647998794000151,
1798
+ "grad_norm": 0.17656107246875763,
1799
+ "learning_rate": 5.276320254727187e-05,
1800
+ "loss": 0.4545,
1801
+ "step": 12800
1802
+ },
1803
+ {
1804
+ "epoch": 0.9685686289289214,
1805
+ "grad_norm": 0.17876358330249786,
1806
+ "learning_rate": 5.2467611308677836e-05,
1807
+ "loss": 0.4566,
1808
+ "step": 12850
1809
+ },
1810
+ {
1811
+ "epoch": 0.9723373784578276,
1812
+ "grad_norm": 0.1829279661178589,
1813
+ "learning_rate": 5.217193359125724e-05,
1814
+ "loss": 0.4562,
1815
+ "step": 12900
1816
+ },
1817
+ {
1818
+ "epoch": 0.976106127986734,
1819
+ "grad_norm": 0.19071832299232483,
1820
+ "learning_rate": 5.187617975720201e-05,
1821
+ "loss": 0.4555,
1822
+ "step": 12950
1823
+ },
1824
+ {
1825
+ "epoch": 0.9798748775156403,
1826
+ "grad_norm": 0.17582501471042633,
1827
+ "learning_rate": 5.15803601713717e-05,
1828
+ "loss": 0.4583,
1829
+ "step": 13000
1830
+ },
1831
+ {
1832
+ "epoch": 0.9836436270445467,
1833
+ "grad_norm": 0.18090075254440308,
1834
+ "learning_rate": 5.1284485200930085e-05,
1835
+ "loss": 0.4555,
1836
+ "step": 13050
1837
+ },
1838
+ {
1839
+ "epoch": 0.9874123765734529,
1840
+ "grad_norm": 0.18311740458011627,
1841
+ "learning_rate": 5.0988565214981976e-05,
1842
+ "loss": 0.4562,
1843
+ "step": 13100
1844
+ },
1845
+ {
1846
+ "epoch": 0.9911811261023592,
1847
+ "grad_norm": 0.18058288097381592,
1848
+ "learning_rate": 5.0692610584209754e-05,
1849
+ "loss": 0.4556,
1850
+ "step": 13150
1851
+ },
1852
+ {
1853
+ "epoch": 0.9949498756312656,
1854
+ "grad_norm": 0.18204964697360992,
1855
+ "learning_rate": 5.0396631680509945e-05,
1856
+ "loss": 0.4548,
1857
+ "step": 13200
1858
+ },
1859
+ {
1860
+ "epoch": 0.9987186251601718,
1861
+ "grad_norm": 0.18574148416519165,
1862
+ "learning_rate": 5.010063887662976e-05,
1863
+ "loss": 0.4514,
1864
+ "step": 13250
1865
+ },
1866
+ {
1867
+ "epoch": 1.0024873746890781,
1868
+ "grad_norm": 0.17709988355636597,
1869
+ "learning_rate": 4.9804642545803524e-05,
1870
+ "loss": 0.4307,
1871
+ "step": 13300
1872
+ },
1873
+ {
1874
+ "epoch": 1.0062561242179844,
1875
+ "grad_norm": 0.18011409044265747,
1876
+ "learning_rate": 4.950865306138917e-05,
1877
+ "loss": 0.4178,
1878
+ "step": 13350
1879
+ },
1880
+ {
1881
+ "epoch": 1.0100248737468909,
1882
+ "grad_norm": 0.18502439558506012,
1883
+ "learning_rate": 4.9212680796504704e-05,
1884
+ "loss": 0.4173,
1885
+ "step": 13400
1886
+ },
1887
+ {
1888
+ "epoch": 1.0137936232757971,
1889
+ "grad_norm": 0.18953478336334229,
1890
+ "learning_rate": 4.8916736123664666e-05,
1891
+ "loss": 0.4197,
1892
+ "step": 13450
1893
+ },
1894
+ {
1895
+ "epoch": 1.0175623728047034,
1896
+ "grad_norm": 0.18180860579013824,
1897
+ "learning_rate": 4.8620829414416615e-05,
1898
+ "loss": 0.4164,
1899
+ "step": 13500
1900
+ },
1901
+ {
1902
+ "epoch": 1.0213311223336097,
1903
+ "grad_norm": 0.1739131063222885,
1904
+ "learning_rate": 4.832497103897762e-05,
1905
+ "loss": 0.4162,
1906
+ "step": 13550
1907
+ },
1908
+ {
1909
+ "epoch": 1.025099871862516,
1910
+ "grad_norm": 0.18233858048915863,
1911
+ "learning_rate": 4.8029171365870926e-05,
1912
+ "loss": 0.4152,
1913
+ "step": 13600
1914
+ },
1915
+ {
1916
+ "epoch": 1.0288686213914224,
1917
+ "grad_norm": 0.18319696187973022,
1918
+ "learning_rate": 4.7733440761562466e-05,
1919
+ "loss": 0.4187,
1920
+ "step": 13650
1921
+ },
1922
+ {
1923
+ "epoch": 1.0326373709203287,
1924
+ "grad_norm": 0.1849289834499359,
1925
+ "learning_rate": 4.743778959009766e-05,
1926
+ "loss": 0.4167,
1927
+ "step": 13700
1928
+ },
1929
+ {
1930
+ "epoch": 1.036406120449235,
1931
+ "grad_norm": 0.17826583981513977,
1932
+ "learning_rate": 4.7142228212738126e-05,
1933
+ "loss": 0.4178,
1934
+ "step": 13750
1935
+ },
1936
+ {
1937
+ "epoch": 1.0401748699781412,
1938
+ "grad_norm": 0.18805526196956635,
1939
+ "learning_rate": 4.684676698759864e-05,
1940
+ "loss": 0.4169,
1941
+ "step": 13800
1942
+ },
1943
+ {
1944
+ "epoch": 1.0439436195070475,
1945
+ "grad_norm": 0.18643540143966675,
1946
+ "learning_rate": 4.655141626928403e-05,
1947
+ "loss": 0.4176,
1948
+ "step": 13850
1949
+ },
1950
+ {
1951
+ "epoch": 1.0477123690359538,
1952
+ "grad_norm": 0.18345114588737488,
1953
+ "learning_rate": 4.62561864085264e-05,
1954
+ "loss": 0.4183,
1955
+ "step": 13900
1956
+ },
1957
+ {
1958
+ "epoch": 1.0514811185648603,
1959
+ "grad_norm": 0.18768061697483063,
1960
+ "learning_rate": 4.596108775182228e-05,
1961
+ "loss": 0.4165,
1962
+ "step": 13950
1963
+ },
1964
+ {
1965
+ "epoch": 1.0552498680937665,
1966
+ "grad_norm": 0.17966210842132568,
1967
+ "learning_rate": 4.566613064107015e-05,
1968
+ "loss": 0.4171,
1969
+ "step": 14000
1970
+ },
1971
+ {
1972
+ "epoch": 1.0590186176226728,
1973
+ "grad_norm": 0.1812521517276764,
1974
+ "learning_rate": 4.5371325413207885e-05,
1975
+ "loss": 0.4191,
1976
+ "step": 14050
1977
+ },
1978
+ {
1979
+ "epoch": 1.062787367151579,
1980
+ "grad_norm": 0.1929241418838501,
1981
+ "learning_rate": 4.507668239985055e-05,
1982
+ "loss": 0.4165,
1983
+ "step": 14100
1984
+ },
1985
+ {
1986
+ "epoch": 1.0665561166804853,
1987
+ "grad_norm": 0.17854027450084686,
1988
+ "learning_rate": 4.478221192692831e-05,
1989
+ "loss": 0.416,
1990
+ "step": 14150
1991
+ },
1992
+ {
1993
+ "epoch": 1.0703248662093918,
1994
+ "grad_norm": 0.18497681617736816,
1995
+ "learning_rate": 4.448792431432451e-05,
1996
+ "loss": 0.4145,
1997
+ "step": 14200
1998
+ },
1999
+ {
2000
+ "epoch": 1.074093615738298,
2001
+ "grad_norm": 0.18206162750720978,
2002
+ "learning_rate": 4.4193829875514176e-05,
2003
+ "loss": 0.4198,
2004
+ "step": 14250
2005
+ },
2006
+ {
2007
+ "epoch": 1.0778623652672044,
2008
+ "grad_norm": 0.17210817337036133,
2009
+ "learning_rate": 4.389993891720232e-05,
2010
+ "loss": 0.4159,
2011
+ "step": 14300
2012
+ },
2013
+ {
2014
+ "epoch": 1.0816311147961106,
2015
+ "grad_norm": 0.17640775442123413,
2016
+ "learning_rate": 4.360626173896293e-05,
2017
+ "loss": 0.4178,
2018
+ "step": 14350
2019
+ },
2020
+ {
2021
+ "epoch": 1.085399864325017,
2022
+ "grad_norm": 0.19096557796001434,
2023
+ "learning_rate": 4.3312808632877924e-05,
2024
+ "loss": 0.4178,
2025
+ "step": 14400
2026
+ },
2027
+ {
2028
+ "epoch": 1.0891686138539232,
2029
+ "grad_norm": 0.18599864840507507,
2030
+ "learning_rate": 4.301958988317651e-05,
2031
+ "loss": 0.4145,
2032
+ "step": 14450
2033
+ },
2034
+ {
2035
+ "epoch": 1.0929373633828297,
2036
+ "grad_norm": 0.17436467111110687,
2037
+ "learning_rate": 4.27266157658747e-05,
2038
+ "loss": 0.4168,
2039
+ "step": 14500
2040
+ },
2041
+ {
2042
+ "epoch": 1.096706112911736,
2043
+ "grad_norm": 0.17950861155986786,
2044
+ "learning_rate": 4.243389654841531e-05,
2045
+ "loss": 0.4169,
2046
+ "step": 14550
2047
+ },
2048
+ {
2049
+ "epoch": 1.1004748624406422,
2050
+ "grad_norm": 0.18077972531318665,
2051
+ "learning_rate": 4.214144248930797e-05,
2052
+ "loss": 0.4152,
2053
+ "step": 14600
2054
+ },
2055
+ {
2056
+ "epoch": 1.1042436119695485,
2057
+ "grad_norm": 0.18444353342056274,
2058
+ "learning_rate": 4.184926383776971e-05,
2059
+ "loss": 0.4154,
2060
+ "step": 14650
2061
+ },
2062
+ {
2063
+ "epoch": 1.1080123614984547,
2064
+ "grad_norm": 0.18004246056079865,
2065
+ "learning_rate": 4.155737083336575e-05,
2066
+ "loss": 0.4155,
2067
+ "step": 14700
2068
+ },
2069
+ {
2070
+ "epoch": 1.1117811110273612,
2071
+ "grad_norm": 0.18254701793193817,
2072
+ "learning_rate": 4.1265773705650646e-05,
2073
+ "loss": 0.4168,
2074
+ "step": 14750
2075
+ },
2076
+ {
2077
+ "epoch": 1.1155498605562675,
2078
+ "grad_norm": 0.18389485776424408,
2079
+ "learning_rate": 4.097448267380979e-05,
2080
+ "loss": 0.4185,
2081
+ "step": 14800
2082
+ },
2083
+ {
2084
+ "epoch": 1.1193186100851737,
2085
+ "grad_norm": 0.17165398597717285,
2086
+ "learning_rate": 4.0683507946301275e-05,
2087
+ "loss": 0.417,
2088
+ "step": 14850
2089
+ },
2090
+ {
2091
+ "epoch": 1.12308735961408,
2092
+ "grad_norm": 0.18042343854904175,
2093
+ "learning_rate": 4.03928597204981e-05,
2094
+ "loss": 0.4152,
2095
+ "step": 14900
2096
+ },
2097
+ {
2098
+ "epoch": 1.1268561091429863,
2099
+ "grad_norm": 0.1767844706773758,
2100
+ "learning_rate": 4.010254818233089e-05,
2101
+ "loss": 0.4164,
2102
+ "step": 14950
2103
+ },
2104
+ {
2105
+ "epoch": 1.1306248586718928,
2106
+ "grad_norm": 0.18272291123867035,
2107
+ "learning_rate": 3.9812583505930786e-05,
2108
+ "loss": 0.4161,
2109
+ "step": 15000
2110
+ },
2111
+ {
2112
+ "epoch": 1.134393608200799,
2113
+ "grad_norm": 0.18290477991104126,
2114
+ "learning_rate": 3.952297585327303e-05,
2115
+ "loss": 0.4173,
2116
+ "step": 15050
2117
+ },
2118
+ {
2119
+ "epoch": 1.1381623577297053,
2120
+ "grad_norm": 0.17782056331634521,
2121
+ "learning_rate": 3.923373537382074e-05,
2122
+ "loss": 0.4143,
2123
+ "step": 15100
2124
+ },
2125
+ {
2126
+ "epoch": 1.1419311072586116,
2127
+ "grad_norm": 0.1805420070886612,
2128
+ "learning_rate": 3.894487220416924e-05,
2129
+ "loss": 0.4146,
2130
+ "step": 15150
2131
+ },
2132
+ {
2133
+ "epoch": 1.1456998567875178,
2134
+ "grad_norm": 0.18226702511310577,
2135
+ "learning_rate": 3.86563964676908e-05,
2136
+ "loss": 0.413,
2137
+ "step": 15200
2138
+ },
2139
+ {
2140
+ "epoch": 1.1494686063164243,
2141
+ "grad_norm": 0.18122869729995728,
2142
+ "learning_rate": 3.836831827417992e-05,
2143
+ "loss": 0.4153,
2144
+ "step": 15250
2145
+ },
2146
+ {
2147
+ "epoch": 1.1532373558453306,
2148
+ "grad_norm": 0.17712771892547607,
2149
+ "learning_rate": 3.808064771949893e-05,
2150
+ "loss": 0.4159,
2151
+ "step": 15300
2152
+ },
2153
+ {
2154
+ "epoch": 1.1570061053742369,
2155
+ "grad_norm": 0.17394891381263733,
2156
+ "learning_rate": 3.77933948852243e-05,
2157
+ "loss": 0.4136,
2158
+ "step": 15350
2159
+ },
2160
+ {
2161
+ "epoch": 1.1607748549031431,
2162
+ "grad_norm": 0.1861298531293869,
2163
+ "learning_rate": 3.75065698382932e-05,
2164
+ "loss": 0.4146,
2165
+ "step": 15400
2166
+ },
2167
+ {
2168
+ "epoch": 1.1645436044320494,
2169
+ "grad_norm": 0.18379145860671997,
2170
+ "learning_rate": 3.722018263065075e-05,
2171
+ "loss": 0.4148,
2172
+ "step": 15450
2173
+ },
2174
+ {
2175
+ "epoch": 1.1683123539609557,
2176
+ "grad_norm": 0.1812155693769455,
2177
+ "learning_rate": 3.693424329889776e-05,
2178
+ "loss": 0.4147,
2179
+ "step": 15500
2180
+ },
2181
+ {
2182
+ "epoch": 1.172081103489862,
2183
+ "grad_norm": 0.1795729398727417,
2184
+ "learning_rate": 3.664876186393897e-05,
2185
+ "loss": 0.4158,
2186
+ "step": 15550
2187
+ },
2188
+ {
2189
+ "epoch": 1.1758498530187684,
2190
+ "grad_norm": 0.18769286572933197,
2191
+ "learning_rate": 3.636374833063191e-05,
2192
+ "loss": 0.4154,
2193
+ "step": 15600
2194
+ },
2195
+ {
2196
+ "epoch": 1.1796186025476747,
2197
+ "grad_norm": 0.18426291644573212,
2198
+ "learning_rate": 3.6079212687436176e-05,
2199
+ "loss": 0.4134,
2200
+ "step": 15650
2201
+ },
2202
+ {
2203
+ "epoch": 1.183387352076581,
2204
+ "grad_norm": 0.18709790706634521,
2205
+ "learning_rate": 3.579516490606346e-05,
2206
+ "loss": 0.4151,
2207
+ "step": 15700
2208
+ },
2209
+ {
2210
+ "epoch": 1.1871561016054872,
2211
+ "grad_norm": 0.18941350281238556,
2212
+ "learning_rate": 3.551161494112807e-05,
2213
+ "loss": 0.413,
2214
+ "step": 15750
2215
+ },
2216
+ {
2217
+ "epoch": 1.1909248511343935,
2218
+ "grad_norm": 0.1875937581062317,
2219
+ "learning_rate": 3.522857272979804e-05,
2220
+ "loss": 0.4135,
2221
+ "step": 15800
2222
+ },
2223
+ {
2224
+ "epoch": 1.1946936006633,
2225
+ "grad_norm": 0.18288429081439972,
2226
+ "learning_rate": 3.494604819144688e-05,
2227
+ "loss": 0.4125,
2228
+ "step": 15850
2229
+ },
2230
+ {
2231
+ "epoch": 1.1984623501922063,
2232
+ "grad_norm": 0.1818443238735199,
2233
+ "learning_rate": 3.4664051227306026e-05,
2234
+ "loss": 0.4114,
2235
+ "step": 15900
2236
+ },
2237
+ {
2238
+ "epoch": 1.2022310997211125,
2239
+ "grad_norm": 0.18611133098602295,
2240
+ "learning_rate": 3.438259172011772e-05,
2241
+ "loss": 0.4143,
2242
+ "step": 15950
2243
+ },
2244
+ {
2245
+ "epoch": 1.2059998492500188,
2246
+ "grad_norm": 0.18321532011032104,
2247
+ "learning_rate": 3.4101679533788734e-05,
2248
+ "loss": 0.4134,
2249
+ "step": 16000
2250
+ },
2251
+ {
2252
+ "epoch": 1.209768598778925,
2253
+ "grad_norm": 0.1766360104084015,
2254
+ "learning_rate": 3.382132451304465e-05,
2255
+ "loss": 0.4127,
2256
+ "step": 16050
2257
+ },
2258
+ {
2259
+ "epoch": 1.2135373483078316,
2260
+ "grad_norm": 0.1815403252840042,
2261
+ "learning_rate": 3.354153648308492e-05,
2262
+ "loss": 0.4118,
2263
+ "step": 16100
2264
+ },
2265
+ {
2266
+ "epoch": 1.2173060978367378,
2267
+ "grad_norm": 0.1798935979604721,
2268
+ "learning_rate": 3.3262325249238466e-05,
2269
+ "loss": 0.4114,
2270
+ "step": 16150
2271
+ },
2272
+ {
2273
+ "epoch": 1.221074847365644,
2274
+ "grad_norm": 0.1812141090631485,
2275
+ "learning_rate": 3.298370059662004e-05,
2276
+ "loss": 0.4131,
2277
+ "step": 16200
2278
+ },
2279
+ {
2280
+ "epoch": 1.2248435968945504,
2281
+ "grad_norm": 0.1809026598930359,
2282
+ "learning_rate": 3.270567228978736e-05,
2283
+ "loss": 0.4124,
2284
+ "step": 16250
2285
+ },
2286
+ {
2287
+ "epoch": 1.2286123464234566,
2288
+ "grad_norm": 0.18148508667945862,
2289
+ "learning_rate": 3.2428250072398846e-05,
2290
+ "loss": 0.4112,
2291
+ "step": 16300
2292
+ },
2293
+ {
2294
+ "epoch": 1.2323810959523631,
2295
+ "grad_norm": 0.18835850059986115,
2296
+ "learning_rate": 3.2151443666872166e-05,
2297
+ "loss": 0.4132,
2298
+ "step": 16350
2299
+ },
2300
+ {
2301
+ "epoch": 1.2361498454812694,
2302
+ "grad_norm": 0.18036408722400665,
2303
+ "learning_rate": 3.187526277404355e-05,
2304
+ "loss": 0.4128,
2305
+ "step": 16400
2306
+ },
2307
+ {
2308
+ "epoch": 1.2399185950101757,
2309
+ "grad_norm": 0.1809609979391098,
2310
+ "learning_rate": 3.159971707282776e-05,
2311
+ "loss": 0.413,
2312
+ "step": 16450
2313
+ },
2314
+ {
2315
+ "epoch": 1.243687344539082,
2316
+ "grad_norm": 0.19554051756858826,
2317
+ "learning_rate": 3.1324816219878903e-05,
2318
+ "loss": 0.4109,
2319
+ "step": 16500
2320
+ },
2321
+ {
2322
+ "epoch": 1.2474560940679882,
2323
+ "grad_norm": 0.1792004555463791,
2324
+ "learning_rate": 3.1050569849252044e-05,
2325
+ "loss": 0.4123,
2326
+ "step": 16550
2327
+ },
2328
+ {
2329
+ "epoch": 1.2512248435968947,
2330
+ "grad_norm": 0.19579829275608063,
2331
+ "learning_rate": 3.077698757206552e-05,
2332
+ "loss": 0.41,
2333
+ "step": 16600
2334
+ },
2335
+ {
2336
+ "epoch": 1.2549935931258007,
2337
+ "grad_norm": 0.18006202578544617,
2338
+ "learning_rate": 3.0504078976164118e-05,
2339
+ "loss": 0.4114,
2340
+ "step": 16650
2341
+ },
2342
+ {
2343
+ "epoch": 1.2587623426547072,
2344
+ "grad_norm": 0.18532243371009827,
2345
+ "learning_rate": 3.0231853625783163e-05,
2346
+ "loss": 0.4117,
2347
+ "step": 16700
2348
+ },
2349
+ {
2350
+ "epoch": 1.2625310921836135,
2351
+ "grad_norm": 0.17538198828697205,
2352
+ "learning_rate": 2.9960321061213193e-05,
2353
+ "loss": 0.4108,
2354
+ "step": 16750
2355
+ },
2356
+ {
2357
+ "epoch": 1.2662998417125197,
2358
+ "grad_norm": 0.1844860017299652,
2359
+ "learning_rate": 2.9689490798465698e-05,
2360
+ "loss": 0.4095,
2361
+ "step": 16800
2362
+ },
2363
+ {
2364
+ "epoch": 1.270068591241426,
2365
+ "grad_norm": 0.18097542226314545,
2366
+ "learning_rate": 2.941937232893959e-05,
2367
+ "loss": 0.4095,
2368
+ "step": 16850
2369
+ },
2370
+ {
2371
+ "epoch": 1.2738373407703323,
2372
+ "grad_norm": 0.1822078973054886,
2373
+ "learning_rate": 2.9149975119088596e-05,
2374
+ "loss": 0.411,
2375
+ "step": 16900
2376
+ },
2377
+ {
2378
+ "epoch": 1.2776060902992388,
2379
+ "grad_norm": 0.18340744078159332,
2380
+ "learning_rate": 2.8881308610089496e-05,
2381
+ "loss": 0.4125,
2382
+ "step": 16950
2383
+ },
2384
+ {
2385
+ "epoch": 1.281374839828145,
2386
+ "grad_norm": 0.18344102799892426,
2387
+ "learning_rate": 2.8613382217511265e-05,
2388
+ "loss": 0.4116,
2389
+ "step": 17000
2390
+ },
2391
+ {
2392
+ "epoch": 1.2851435893570513,
2393
+ "grad_norm": 0.18103405833244324,
2394
+ "learning_rate": 2.8346205330985053e-05,
2395
+ "loss": 0.4099,
2396
+ "step": 17050
2397
+ },
2398
+ {
2399
+ "epoch": 1.2889123388859576,
2400
+ "grad_norm": 0.18688762187957764,
2401
+ "learning_rate": 2.807978731387516e-05,
2402
+ "loss": 0.4113,
2403
+ "step": 17100
2404
+ },
2405
+ {
2406
+ "epoch": 1.2926810884148638,
2407
+ "grad_norm": 0.17642515897750854,
2408
+ "learning_rate": 2.7814137502950878e-05,
2409
+ "loss": 0.4102,
2410
+ "step": 17150
2411
+ },
2412
+ {
2413
+ "epoch": 1.2964498379437703,
2414
+ "grad_norm": 0.1789291650056839,
2415
+ "learning_rate": 2.754926520805925e-05,
2416
+ "loss": 0.409,
2417
+ "step": 17200
2418
+ },
2419
+ {
2420
+ "epoch": 1.3002185874726766,
2421
+ "grad_norm": 0.1897159218788147,
2422
+ "learning_rate": 2.728517971179892e-05,
2423
+ "loss": 0.4095,
2424
+ "step": 17250
2425
+ },
2426
+ {
2427
+ "epoch": 1.3039873370015829,
2428
+ "grad_norm": 0.1783190369606018,
2429
+ "learning_rate": 2.702189026919465e-05,
2430
+ "loss": 0.4094,
2431
+ "step": 17300
2432
+ },
2433
+ {
2434
+ "epoch": 1.3077560865304891,
2435
+ "grad_norm": 0.18828634917736053,
2436
+ "learning_rate": 2.675940610737307e-05,
2437
+ "loss": 0.4068,
2438
+ "step": 17350
2439
+ },
2440
+ {
2441
+ "epoch": 1.3115248360593954,
2442
+ "grad_norm": 0.19085575640201569,
2443
+ "learning_rate": 2.6497736425239315e-05,
2444
+ "loss": 0.4098,
2445
+ "step": 17400
2446
+ },
2447
+ {
2448
+ "epoch": 1.315293585588302,
2449
+ "grad_norm": 0.1795608252286911,
2450
+ "learning_rate": 2.623689039315458e-05,
2451
+ "loss": 0.4086,
2452
+ "step": 17450
2453
+ },
2454
+ {
2455
+ "epoch": 1.3190623351172082,
2456
+ "grad_norm": 0.17982016503810883,
2457
+ "learning_rate": 2.597687715261484e-05,
2458
+ "loss": 0.4077,
2459
+ "step": 17500
2460
+ },
2461
+ {
2462
+ "epoch": 1.3228310846461144,
2463
+ "grad_norm": 0.17843079566955566,
2464
+ "learning_rate": 2.5717705815930392e-05,
2465
+ "loss": 0.4097,
2466
+ "step": 17550
2467
+ },
2468
+ {
2469
+ "epoch": 1.3265998341750207,
2470
+ "grad_norm": 0.1862371265888214,
2471
+ "learning_rate": 2.5459385465906517e-05,
2472
+ "loss": 0.4078,
2473
+ "step": 17600
2474
+ },
2475
+ {
2476
+ "epoch": 1.330368583703927,
2477
+ "grad_norm": 0.18295156955718994,
2478
+ "learning_rate": 2.520192515552522e-05,
2479
+ "loss": 0.4093,
2480
+ "step": 17650
2481
+ },
2482
+ {
2483
+ "epoch": 1.3341373332328335,
2484
+ "grad_norm": 0.19080038368701935,
2485
+ "learning_rate": 2.4945333907627892e-05,
2486
+ "loss": 0.4067,
2487
+ "step": 17700
2488
+ },
2489
+ {
2490
+ "epoch": 1.3379060827617397,
2491
+ "grad_norm": 0.17629754543304443,
2492
+ "learning_rate": 2.468962071459922e-05,
2493
+ "loss": 0.4099,
2494
+ "step": 17750
2495
+ },
2496
+ {
2497
+ "epoch": 1.341674832290646,
2498
+ "grad_norm": 0.18612775206565857,
2499
+ "learning_rate": 2.443479453805189e-05,
2500
+ "loss": 0.4084,
2501
+ "step": 17800
2502
+ },
2503
+ {
2504
+ "epoch": 1.3454435818195523,
2505
+ "grad_norm": 0.17433632910251617,
2506
+ "learning_rate": 2.4180864308512624e-05,
2507
+ "loss": 0.407,
2508
+ "step": 17850
2509
+ },
2510
+ {
2511
+ "epoch": 1.3492123313484585,
2512
+ "grad_norm": 0.17874571681022644,
2513
+ "learning_rate": 2.392783892510917e-05,
2514
+ "loss": 0.4075,
2515
+ "step": 17900
2516
+ },
2517
+ {
2518
+ "epoch": 1.352981080877365,
2519
+ "grad_norm": 0.18273356556892395,
2520
+ "learning_rate": 2.367572725525844e-05,
2521
+ "loss": 0.4063,
2522
+ "step": 17950
2523
+ },
2524
+ {
2525
+ "epoch": 1.356749830406271,
2526
+ "grad_norm": 0.17750360071659088,
2527
+ "learning_rate": 2.3424538134355715e-05,
2528
+ "loss": 0.4075,
2529
+ "step": 18000
2530
+ },
2531
+ {
2532
+ "epoch": 1.3605185799351776,
2533
+ "grad_norm": 0.17407415807247162,
2534
+ "learning_rate": 2.3174280365465102e-05,
2535
+ "loss": 0.4062,
2536
+ "step": 18050
2537
+ },
2538
+ {
2539
+ "epoch": 1.3642873294640838,
2540
+ "grad_norm": 0.19176089763641357,
2541
+ "learning_rate": 2.2924962719010874e-05,
2542
+ "loss": 0.4068,
2543
+ "step": 18100
2544
+ },
2545
+ {
2546
+ "epoch": 1.36805607899299,
2547
+ "grad_norm": 0.18244892358779907,
2548
+ "learning_rate": 2.2676593932470213e-05,
2549
+ "loss": 0.4072,
2550
+ "step": 18150
2551
+ },
2552
+ {
2553
+ "epoch": 1.3718248285218964,
2554
+ "grad_norm": 0.18260513246059418,
2555
+ "learning_rate": 2.242918271006698e-05,
2556
+ "loss": 0.4079,
2557
+ "step": 18200
2558
+ },
2559
+ {
2560
+ "epoch": 1.3755935780508026,
2561
+ "grad_norm": 0.18689802289009094,
2562
+ "learning_rate": 2.2182737722466633e-05,
2563
+ "loss": 0.4045,
2564
+ "step": 18250
2565
+ },
2566
+ {
2567
+ "epoch": 1.3793623275797091,
2568
+ "grad_norm": 0.1868721842765808,
2569
+ "learning_rate": 2.193726760647245e-05,
2570
+ "loss": 0.4061,
2571
+ "step": 18300
2572
+ },
2573
+ {
2574
+ "epoch": 1.3831310771086154,
2575
+ "grad_norm": 0.18145039677619934,
2576
+ "learning_rate": 2.169278096472272e-05,
2577
+ "loss": 0.4062,
2578
+ "step": 18350
2579
+ },
2580
+ {
2581
+ "epoch": 1.3868998266375216,
2582
+ "grad_norm": 0.18034633994102478,
2583
+ "learning_rate": 2.1449286365389342e-05,
2584
+ "loss": 0.4051,
2585
+ "step": 18400
2586
+ },
2587
+ {
2588
+ "epoch": 1.390668576166428,
2589
+ "grad_norm": 0.1792653501033783,
2590
+ "learning_rate": 2.120679234187755e-05,
2591
+ "loss": 0.406,
2592
+ "step": 18450
2593
+ },
2594
+ {
2595
+ "epoch": 1.3944373256953342,
2596
+ "grad_norm": 0.17878781259059906,
2597
+ "learning_rate": 2.0965307392526818e-05,
2598
+ "loss": 0.4075,
2599
+ "step": 18500
2600
+ },
2601
+ {
2602
+ "epoch": 1.3982060752242407,
2603
+ "grad_norm": 0.18702946603298187,
2604
+ "learning_rate": 2.072483998031303e-05,
2605
+ "loss": 0.4067,
2606
+ "step": 18550
2607
+ },
2608
+ {
2609
+ "epoch": 1.401974824753147,
2610
+ "grad_norm": 0.1810487061738968,
2611
+ "learning_rate": 2.048539853255197e-05,
2612
+ "loss": 0.4045,
2613
+ "step": 18600
2614
+ },
2615
+ {
2616
+ "epoch": 1.4057435742820532,
2617
+ "grad_norm": 0.17860865592956543,
2618
+ "learning_rate": 2.0246991440603862e-05,
2619
+ "loss": 0.405,
2620
+ "step": 18650
2621
+ },
2622
+ {
2623
+ "epoch": 1.4095123238109595,
2624
+ "grad_norm": 0.1859513223171234,
2625
+ "learning_rate": 2.0009627059579372e-05,
2626
+ "loss": 0.4058,
2627
+ "step": 18700
2628
+ },
2629
+ {
2630
+ "epoch": 1.4132810733398657,
2631
+ "grad_norm": 0.18461273610591888,
2632
+ "learning_rate": 1.9773313708046758e-05,
2633
+ "loss": 0.4054,
2634
+ "step": 18750
2635
+ },
2636
+ {
2637
+ "epoch": 1.4170498228687722,
2638
+ "grad_norm": 0.18574093282222748,
2639
+ "learning_rate": 1.953805966774037e-05,
2640
+ "loss": 0.406,
2641
+ "step": 18800
2642
+ },
2643
+ {
2644
+ "epoch": 1.4208185723976785,
2645
+ "grad_norm": 0.1719568520784378,
2646
+ "learning_rate": 1.930387318327043e-05,
2647
+ "loss": 0.4056,
2648
+ "step": 18850
2649
+ },
2650
+ {
2651
+ "epoch": 1.4245873219265848,
2652
+ "grad_norm": 0.18191532790660858,
2653
+ "learning_rate": 1.9070762461834018e-05,
2654
+ "loss": 0.4045,
2655
+ "step": 18900
2656
+ },
2657
+ {
2658
+ "epoch": 1.428356071455491,
2659
+ "grad_norm": 0.1849079728126526,
2660
+ "learning_rate": 1.8838735672927516e-05,
2661
+ "loss": 0.4057,
2662
+ "step": 18950
2663
+ },
2664
+ {
2665
+ "epoch": 1.4321248209843973,
2666
+ "grad_norm": 0.18583734333515167,
2667
+ "learning_rate": 1.8607800948060266e-05,
2668
+ "loss": 0.404,
2669
+ "step": 19000
2670
+ },
2671
+ {
2672
+ "epoch": 1.4358935705133038,
2673
+ "grad_norm": 0.18024706840515137,
2674
+ "learning_rate": 1.837796638046962e-05,
2675
+ "loss": 0.4049,
2676
+ "step": 19050
2677
+ },
2678
+ {
2679
+ "epoch": 1.43966232004221,
2680
+ "grad_norm": 0.18181918561458588,
2681
+ "learning_rate": 1.8149240024837315e-05,
2682
+ "loss": 0.4048,
2683
+ "step": 19100
2684
+ },
2685
+ {
2686
+ "epoch": 1.4434310695711163,
2687
+ "grad_norm": 0.18205614387989044,
2688
+ "learning_rate": 1.792162989700717e-05,
2689
+ "loss": 0.4048,
2690
+ "step": 19150
2691
+ },
2692
+ {
2693
+ "epoch": 1.4471998191000226,
2694
+ "grad_norm": 0.18339186906814575,
2695
+ "learning_rate": 1.7695143973704143e-05,
2696
+ "loss": 0.4065,
2697
+ "step": 19200
2698
+ },
2699
+ {
2700
+ "epoch": 1.4509685686289289,
2701
+ "grad_norm": 0.17691025137901306,
2702
+ "learning_rate": 1.746979019225483e-05,
2703
+ "loss": 0.4042,
2704
+ "step": 19250
2705
+ },
2706
+ {
2707
+ "epoch": 1.4547373181578354,
2708
+ "grad_norm": 0.18054354190826416,
2709
+ "learning_rate": 1.7245576450309316e-05,
2710
+ "loss": 0.4044,
2711
+ "step": 19300
2712
+ },
2713
+ {
2714
+ "epoch": 1.4585060676867414,
2715
+ "grad_norm": 0.17796145379543304,
2716
+ "learning_rate": 1.7022510605564307e-05,
2717
+ "loss": 0.405,
2718
+ "step": 19350
2719
+ },
2720
+ {
2721
+ "epoch": 1.462274817215648,
2722
+ "grad_norm": 0.17998959124088287,
2723
+ "learning_rate": 1.6800600475487826e-05,
2724
+ "loss": 0.4033,
2725
+ "step": 19400
2726
+ },
2727
+ {
2728
+ "epoch": 1.4660435667445542,
2729
+ "grad_norm": 0.18134063482284546,
2730
+ "learning_rate": 1.657985383704521e-05,
2731
+ "loss": 0.4033,
2732
+ "step": 19450
2733
+ },
2734
+ {
2735
+ "epoch": 1.4698123162734604,
2736
+ "grad_norm": 0.18283304572105408,
2737
+ "learning_rate": 1.6360278426426624e-05,
2738
+ "loss": 0.4037,
2739
+ "step": 19500
2740
+ },
2741
+ {
2742
+ "epoch": 1.4735810658023667,
2743
+ "grad_norm": 0.17472119629383087,
2744
+ "learning_rate": 1.6141881938775848e-05,
2745
+ "loss": 0.4045,
2746
+ "step": 19550
2747
+ },
2748
+ {
2749
+ "epoch": 1.477349815331273,
2750
+ "grad_norm": 0.18283599615097046,
2751
+ "learning_rate": 1.5924672027920663e-05,
2752
+ "loss": 0.4036,
2753
+ "step": 19600
2754
+ },
2755
+ {
2756
+ "epoch": 1.4811185648601795,
2757
+ "grad_norm": 0.18116436898708344,
2758
+ "learning_rate": 1.570865630610457e-05,
2759
+ "loss": 0.4034,
2760
+ "step": 19650
2761
+ },
2762
+ {
2763
+ "epoch": 1.4848873143890857,
2764
+ "grad_norm": 0.1848943680524826,
2765
+ "learning_rate": 1.5493842343720104e-05,
2766
+ "loss": 0.4041,
2767
+ "step": 19700
2768
+ },
2769
+ {
2770
+ "epoch": 1.488656063917992,
2771
+ "grad_norm": 0.1797831803560257,
2772
+ "learning_rate": 1.528023766904341e-05,
2773
+ "loss": 0.4035,
2774
+ "step": 19750
2775
+ },
2776
+ {
2777
+ "epoch": 1.4924248134468983,
2778
+ "grad_norm": 0.17644710838794708,
2779
+ "learning_rate": 1.5067849767970488e-05,
2780
+ "loss": 0.4055,
2781
+ "step": 19800
2782
+ },
2783
+ {
2784
+ "epoch": 1.4961935629758045,
2785
+ "grad_norm": 0.18384455144405365,
2786
+ "learning_rate": 1.4856686083754817e-05,
2787
+ "loss": 0.4029,
2788
+ "step": 19850
2789
+ },
2790
+ {
2791
+ "epoch": 1.499962312504711,
2792
+ "grad_norm": 0.18589411675930023,
2793
+ "learning_rate": 1.4646754016746483e-05,
2794
+ "loss": 0.4025,
2795
+ "step": 19900
2796
+ },
2797
+ {
2798
+ "epoch": 1.5037310620336173,
2799
+ "grad_norm": 0.1780794858932495,
2800
+ "learning_rate": 1.4438060924132924e-05,
2801
+ "loss": 0.4027,
2802
+ "step": 19950
2803
+ },
2804
+ {
2805
+ "epoch": 1.5074998115625236,
2806
+ "grad_norm": 0.17796076834201813,
2807
+ "learning_rate": 1.4230614119680957e-05,
2808
+ "loss": 0.4005,
2809
+ "step": 20000
2810
+ },
2811
+ {
2812
+ "epoch": 1.5112685610914298,
2813
+ "grad_norm": 0.18531276285648346,
2814
+ "learning_rate": 1.4024420873480549e-05,
2815
+ "loss": 0.4043,
2816
+ "step": 20050
2817
+ },
2818
+ {
2819
+ "epoch": 1.515037310620336,
2820
+ "grad_norm": 0.17742446064949036,
2821
+ "learning_rate": 1.3819488411690018e-05,
2822
+ "loss": 0.4033,
2823
+ "step": 20100
2824
+ },
2825
+ {
2826
+ "epoch": 1.5188060601492426,
2827
+ "grad_norm": 0.17753294110298157,
2828
+ "learning_rate": 1.361582391628276e-05,
2829
+ "loss": 0.4019,
2830
+ "step": 20150
2831
+ },
2832
+ {
2833
+ "epoch": 1.5225748096781488,
2834
+ "grad_norm": 0.1751794070005417,
2835
+ "learning_rate": 1.3413434524795631e-05,
2836
+ "loss": 0.4012,
2837
+ "step": 20200
2838
+ },
2839
+ {
2840
+ "epoch": 1.5263435592070551,
2841
+ "grad_norm": 0.1775294840335846,
2842
+ "learning_rate": 1.32123273300787e-05,
2843
+ "loss": 0.4029,
2844
+ "step": 20250
2845
+ },
2846
+ {
2847
+ "epoch": 1.5301123087359614,
2848
+ "grad_norm": 0.18354074656963348,
2849
+ "learning_rate": 1.3012509380046745e-05,
2850
+ "loss": 0.4016,
2851
+ "step": 20300
2852
+ },
2853
+ {
2854
+ "epoch": 1.5338810582648676,
2855
+ "grad_norm": 0.19483138620853424,
2856
+ "learning_rate": 1.2813987677432242e-05,
2857
+ "loss": 0.4021,
2858
+ "step": 20350
2859
+ },
2860
+ {
2861
+ "epoch": 1.5376498077937741,
2862
+ "grad_norm": 0.1839035302400589,
2863
+ "learning_rate": 1.2616769179539944e-05,
2864
+ "loss": 0.4021,
2865
+ "step": 20400
2866
+ },
2867
+ {
2868
+ "epoch": 1.5414185573226802,
2869
+ "grad_norm": 0.1727752387523651,
2870
+ "learning_rate": 1.2420860798003047e-05,
2871
+ "loss": 0.4008,
2872
+ "step": 20450
2873
+ },
2874
+ {
2875
+ "epoch": 1.5451873068515867,
2876
+ "grad_norm": 0.1786087602376938,
2877
+ "learning_rate": 1.222626939854103e-05,
2878
+ "loss": 0.4008,
2879
+ "step": 20500
2880
+ },
2881
+ {
2882
+ "epoch": 1.548956056380493,
2883
+ "grad_norm": 0.17971089482307434,
2884
+ "learning_rate": 1.203300180071894e-05,
2885
+ "loss": 0.4003,
2886
+ "step": 20550
2887
+ },
2888
+ {
2889
+ "epoch": 1.5527248059093992,
2890
+ "grad_norm": 0.17970240116119385,
2891
+ "learning_rate": 1.1841064777708483e-05,
2892
+ "loss": 0.4025,
2893
+ "step": 20600
2894
+ },
2895
+ {
2896
+ "epoch": 1.5564935554383057,
2897
+ "grad_norm": 0.1815640926361084,
2898
+ "learning_rate": 1.1650465056050597e-05,
2899
+ "loss": 0.4029,
2900
+ "step": 20650
2901
+ },
2902
+ {
2903
+ "epoch": 1.5602623049672117,
2904
+ "grad_norm": 0.18187712132930756,
2905
+ "learning_rate": 1.1461209315419758e-05,
2906
+ "loss": 0.4014,
2907
+ "step": 20700
2908
+ },
2909
+ {
2910
+ "epoch": 1.5640310544961182,
2911
+ "grad_norm": 0.18395179510116577,
2912
+ "learning_rate": 1.1273304188389882e-05,
2913
+ "loss": 0.4022,
2914
+ "step": 20750
2915
+ },
2916
+ {
2917
+ "epoch": 1.5677998040250245,
2918
+ "grad_norm": 0.18109290301799774,
2919
+ "learning_rate": 1.1086756260201859e-05,
2920
+ "loss": 0.4001,
2921
+ "step": 20800
2922
+ },
2923
+ {
2924
+ "epoch": 1.5715685535539308,
2925
+ "grad_norm": 0.17941805720329285,
2926
+ "learning_rate": 1.0901572068532773e-05,
2927
+ "loss": 0.4017,
2928
+ "step": 20850
2929
+ },
2930
+ {
2931
+ "epoch": 1.5753373030828373,
2932
+ "grad_norm": 0.1812305897474289,
2933
+ "learning_rate": 1.0717758103266805e-05,
2934
+ "loss": 0.4003,
2935
+ "step": 20900
2936
+ },
2937
+ {
2938
+ "epoch": 1.5791060526117433,
2939
+ "grad_norm": 0.1804351657629013,
2940
+ "learning_rate": 1.0535320806267767e-05,
2941
+ "loss": 0.4007,
2942
+ "step": 20950
2943
+ },
2944
+ {
2945
+ "epoch": 1.5828748021406498,
2946
+ "grad_norm": 0.1748201847076416,
2947
+ "learning_rate": 1.0354266571153399e-05,
2948
+ "loss": 0.4004,
2949
+ "step": 21000
2950
+ },
2951
+ {
2952
+ "epoch": 1.586643551669556,
2953
+ "grad_norm": 0.18150129914283752,
2954
+ "learning_rate": 1.0174601743071205e-05,
2955
+ "loss": 0.402,
2956
+ "step": 21050
2957
+ },
2958
+ {
2959
+ "epoch": 1.5904123011984623,
2960
+ "grad_norm": 0.175798237323761,
2961
+ "learning_rate": 9.996332618476172e-06,
2962
+ "loss": 0.402,
2963
+ "step": 21100
2964
+ },
2965
+ {
2966
+ "epoch": 1.5941810507273688,
2967
+ "grad_norm": 0.1742759644985199,
2968
+ "learning_rate": 9.819465444910058e-06,
2969
+ "loss": 0.4001,
2970
+ "step": 21150
2971
+ },
2972
+ {
2973
+ "epoch": 1.5979498002562749,
2974
+ "grad_norm": 0.19138532876968384,
2975
+ "learning_rate": 9.644006420782476e-06,
2976
+ "loss": 0.4002,
2977
+ "step": 21200
2978
+ },
2979
+ {
2980
+ "epoch": 1.6017185497851814,
2981
+ "grad_norm": 0.18248885869979858,
2982
+ "learning_rate": 9.469961695153606e-06,
2983
+ "loss": 0.4018,
2984
+ "step": 21250
2985
+ },
2986
+ {
2987
+ "epoch": 1.6054872993140876,
2988
+ "grad_norm": 0.1761186271905899,
2989
+ "learning_rate": 9.29733736751881e-06,
2990
+ "loss": 0.4027,
2991
+ "step": 21300
2992
+ },
2993
+ {
2994
+ "epoch": 1.609256048842994,
2995
+ "grad_norm": 0.17524461448192596,
2996
+ "learning_rate": 9.126139487594749e-06,
2997
+ "loss": 0.4,
2998
+ "step": 21350
2999
+ },
3000
+ {
3001
+ "epoch": 1.6130247983719002,
3002
+ "grad_norm": 0.18030941486358643,
3003
+ "learning_rate": 8.956374055107442e-06,
3004
+ "loss": 0.4008,
3005
+ "step": 21400
3006
+ },
3007
+ {
3008
+ "epoch": 1.6167935479008064,
3009
+ "grad_norm": 0.17748509347438812,
3010
+ "learning_rate": 8.78804701958198e-06,
3011
+ "loss": 0.3987,
3012
+ "step": 21450
3013
+ },
3014
+ {
3015
+ "epoch": 1.620562297429713,
3016
+ "grad_norm": 0.180355504155159,
3017
+ "learning_rate": 8.621164280134004e-06,
3018
+ "loss": 0.4024,
3019
+ "step": 21500
3020
+ },
3021
+ {
3022
+ "epoch": 1.6243310469586192,
3023
+ "grad_norm": 0.1831391304731369,
3024
+ "learning_rate": 8.45573168526303e-06,
3025
+ "loss": 0.3987,
3026
+ "step": 21550
3027
+ },
3028
+ {
3029
+ "epoch": 1.6280997964875255,
3030
+ "grad_norm": 0.1812552660703659,
3031
+ "learning_rate": 8.291755032647402e-06,
3032
+ "loss": 0.3981,
3033
+ "step": 21600
3034
+ },
3035
+ {
3036
+ "epoch": 1.6318685460164317,
3037
+ "grad_norm": 0.18903885781764984,
3038
+ "learning_rate": 8.129240068941163e-06,
3039
+ "loss": 0.3995,
3040
+ "step": 21650
3041
+ },
3042
+ {
3043
+ "epoch": 1.635637295545338,
3044
+ "grad_norm": 0.18726922571659088,
3045
+ "learning_rate": 7.96819248957265e-06,
3046
+ "loss": 0.4003,
3047
+ "step": 21700
3048
+ },
3049
+ {
3050
+ "epoch": 1.6394060450742445,
3051
+ "grad_norm": 0.18688932061195374,
3052
+ "learning_rate": 7.808617938544866e-06,
3053
+ "loss": 0.3972,
3054
+ "step": 21750
3055
+ },
3056
+ {
3057
+ "epoch": 1.6431747946031505,
3058
+ "grad_norm": 0.17523853480815887,
3059
+ "learning_rate": 7.650522008237754e-06,
3060
+ "loss": 0.4016,
3061
+ "step": 21800
3062
+ },
3063
+ {
3064
+ "epoch": 1.646943544132057,
3065
+ "grad_norm": 0.1828313022851944,
3066
+ "learning_rate": 7.493910239212121e-06,
3067
+ "loss": 0.4014,
3068
+ "step": 21850
3069
+ },
3070
+ {
3071
+ "epoch": 1.6507122936609633,
3072
+ "grad_norm": 0.18200884759426117,
3073
+ "learning_rate": 7.338788120015522e-06,
3074
+ "loss": 0.3977,
3075
+ "step": 21900
3076
+ },
3077
+ {
3078
+ "epoch": 1.6544810431898695,
3079
+ "grad_norm": 0.1785590499639511,
3080
+ "learning_rate": 7.185161086989883e-06,
3081
+ "loss": 0.3981,
3082
+ "step": 21950
3083
+ },
3084
+ {
3085
+ "epoch": 1.658249792718776,
3086
+ "grad_norm": 0.17938192188739777,
3087
+ "learning_rate": 7.033034524081023e-06,
3088
+ "loss": 0.3986,
3089
+ "step": 22000
3090
+ },
3091
+ {
3092
+ "epoch": 1.662018542247682,
3093
+ "grad_norm": 0.18355585634708405,
3094
+ "learning_rate": 6.882413762649914e-06,
3095
+ "loss": 0.3996,
3096
+ "step": 22050
3097
+ },
3098
+ {
3099
+ "epoch": 1.6657872917765886,
3100
+ "grad_norm": 0.17709852755069733,
3101
+ "learning_rate": 6.733304081285874e-06,
3102
+ "loss": 0.3994,
3103
+ "step": 22100
3104
+ },
3105
+ {
3106
+ "epoch": 1.6695560413054948,
3107
+ "grad_norm": 0.1820652186870575,
3108
+ "learning_rate": 6.585710705621584e-06,
3109
+ "loss": 0.3991,
3110
+ "step": 22150
3111
+ },
3112
+ {
3113
+ "epoch": 1.673324790834401,
3114
+ "grad_norm": 0.17669349908828735,
3115
+ "learning_rate": 6.439638808149923e-06,
3116
+ "loss": 0.4008,
3117
+ "step": 22200
3118
+ },
3119
+ {
3120
+ "epoch": 1.6770935403633076,
3121
+ "grad_norm": 0.17596611380577087,
3122
+ "learning_rate": 6.295093508042716e-06,
3123
+ "loss": 0.3995,
3124
+ "step": 22250
3125
+ },
3126
+ {
3127
+ "epoch": 1.6808622898922136,
3128
+ "grad_norm": 0.18253453075885773,
3129
+ "learning_rate": 6.152079870971311e-06,
3130
+ "loss": 0.3971,
3131
+ "step": 22300
3132
+ },
3133
+ {
3134
+ "epoch": 1.6846310394211201,
3135
+ "grad_norm": 0.1786227822303772,
3136
+ "learning_rate": 6.0106029089291025e-06,
3137
+ "loss": 0.3983,
3138
+ "step": 22350
3139
+ },
3140
+ {
3141
+ "epoch": 1.6883997889500264,
3142
+ "grad_norm": 0.18454194068908691,
3143
+ "learning_rate": 5.870667580055805e-06,
3144
+ "loss": 0.3999,
3145
+ "step": 22400
3146
+ },
3147
+ {
3148
+ "epoch": 1.6921685384789327,
3149
+ "grad_norm": 0.17887701094150543,
3150
+ "learning_rate": 5.732278788463746e-06,
3151
+ "loss": 0.3971,
3152
+ "step": 22450
3153
+ },
3154
+ {
3155
+ "epoch": 1.6959372880078392,
3156
+ "grad_norm": 0.18452878296375275,
3157
+ "learning_rate": 5.595441384065986e-06,
3158
+ "loss": 0.3975,
3159
+ "step": 22500
3160
+ },
3161
+ {
3162
+ "epoch": 1.6997060375367452,
3163
+ "grad_norm": 0.17868489027023315,
3164
+ "learning_rate": 5.460160162406336e-06,
3165
+ "loss": 0.4006,
3166
+ "step": 22550
3167
+ },
3168
+ {
3169
+ "epoch": 1.7034747870656517,
3170
+ "grad_norm": 0.17659594118595123,
3171
+ "learning_rate": 5.3264398644913114e-06,
3172
+ "loss": 0.3974,
3173
+ "step": 22600
3174
+ },
3175
+ {
3176
+ "epoch": 1.707243536594558,
3177
+ "grad_norm": 0.17893588542938232,
3178
+ "learning_rate": 5.194285176624003e-06,
3179
+ "loss": 0.3983,
3180
+ "step": 22650
3181
+ },
3182
+ {
3183
+ "epoch": 1.7110122861234642,
3184
+ "grad_norm": 0.17637696862220764,
3185
+ "learning_rate": 5.063700730239784e-06,
3186
+ "loss": 0.3977,
3187
+ "step": 22700
3188
+ },
3189
+ {
3190
+ "epoch": 1.7147810356523705,
3191
+ "grad_norm": 0.17973153293132782,
3192
+ "learning_rate": 4.934691101744043e-06,
3193
+ "loss": 0.3977,
3194
+ "step": 22750
3195
+ },
3196
+ {
3197
+ "epoch": 1.7185497851812768,
3198
+ "grad_norm": 0.1788349598646164,
3199
+ "learning_rate": 4.807260812351793e-06,
3200
+ "loss": 0.3967,
3201
+ "step": 22800
3202
+ },
3203
+ {
3204
+ "epoch": 1.7223185347101833,
3205
+ "grad_norm": 0.1776108741760254,
3206
+ "learning_rate": 4.681414327929201e-06,
3207
+ "loss": 0.4005,
3208
+ "step": 22850
3209
+ },
3210
+ {
3211
+ "epoch": 1.7260872842390895,
3212
+ "grad_norm": 0.18279901146888733,
3213
+ "learning_rate": 4.557156058837137e-06,
3214
+ "loss": 0.3968,
3215
+ "step": 22900
3216
+ },
3217
+ {
3218
+ "epoch": 1.7298560337679958,
3219
+ "grad_norm": 0.18151997029781342,
3220
+ "learning_rate": 4.4344903597765386e-06,
3221
+ "loss": 0.4007,
3222
+ "step": 22950
3223
+ },
3224
+ {
3225
+ "epoch": 1.733624783296902,
3226
+ "grad_norm": 0.17980162799358368,
3227
+ "learning_rate": 4.31342152963583e-06,
3228
+ "loss": 0.3978,
3229
+ "step": 23000
3230
+ },
3231
+ {
3232
+ "epoch": 1.7373935328258083,
3233
+ "grad_norm": 0.1825917512178421,
3234
+ "learning_rate": 4.19395381134029e-06,
3235
+ "loss": 0.3984,
3236
+ "step": 23050
3237
+ },
3238
+ {
3239
+ "epoch": 1.7411622823547148,
3240
+ "grad_norm": 0.17702624201774597,
3241
+ "learning_rate": 4.076091391703302e-06,
3242
+ "loss": 0.3996,
3243
+ "step": 23100
3244
+ },
3245
+ {
3246
+ "epoch": 1.7449310318836209,
3247
+ "grad_norm": 0.18847475945949554,
3248
+ "learning_rate": 3.9598384012796755e-06,
3249
+ "loss": 0.3985,
3250
+ "step": 23150
3251
+ },
3252
+ {
3253
+ "epoch": 1.7486997814125274,
3254
+ "grad_norm": 0.17738161981105804,
3255
+ "learning_rate": 3.845198914220871e-06,
3256
+ "loss": 0.396,
3257
+ "step": 23200
3258
+ },
3259
+ {
3260
+ "epoch": 1.7524685309414336,
3261
+ "grad_norm": 0.17686378955841064,
3262
+ "learning_rate": 3.732176948132199e-06,
3263
+ "loss": 0.3968,
3264
+ "step": 23250
3265
+ },
3266
+ {
3267
+ "epoch": 1.75623728047034,
3268
+ "grad_norm": 0.17942722141742706,
3269
+ "learning_rate": 3.6207764639320462e-06,
3270
+ "loss": 0.3973,
3271
+ "step": 23300
3272
+ },
3273
+ {
3274
+ "epoch": 1.7600060299992464,
3275
+ "grad_norm": 0.17456026375293732,
3276
+ "learning_rate": 3.511001365713057e-06,
3277
+ "loss": 0.3966,
3278
+ "step": 23350
3279
+ },
3280
+ {
3281
+ "epoch": 1.7637747795281524,
3282
+ "grad_norm": 0.17901940643787384,
3283
+ "learning_rate": 3.4028555006052953e-06,
3284
+ "loss": 0.3958,
3285
+ "step": 23400
3286
+ },
3287
+ {
3288
+ "epoch": 1.767543529057059,
3289
+ "grad_norm": 0.1767757087945938,
3290
+ "learning_rate": 3.2963426586414657e-06,
3291
+ "loss": 0.3976,
3292
+ "step": 23450
3293
+ },
3294
+ {
3295
+ "epoch": 1.7713122785859652,
3296
+ "grad_norm": 0.18666401505470276,
3297
+ "learning_rate": 3.191466572624019e-06,
3298
+ "loss": 0.3952,
3299
+ "step": 23500
3300
+ },
3301
+ {
3302
+ "epoch": 1.7750810281148715,
3303
+ "grad_norm": 0.17169831693172455,
3304
+ "learning_rate": 3.088230917994406e-06,
3305
+ "loss": 0.3964,
3306
+ "step": 23550
3307
+ },
3308
+ {
3309
+ "epoch": 1.778849777643778,
3310
+ "grad_norm": 0.18045750260353088,
3311
+ "learning_rate": 2.986639312704209e-06,
3312
+ "loss": 0.3949,
3313
+ "step": 23600
3314
+ },
3315
+ {
3316
+ "epoch": 1.782618527172684,
3317
+ "grad_norm": 0.17623819410800934,
3318
+ "learning_rate": 2.8866953170883948e-06,
3319
+ "loss": 0.396,
3320
+ "step": 23650
3321
+ },
3322
+ {
3323
+ "epoch": 1.7863872767015905,
3324
+ "grad_norm": 0.1810029298067093,
3325
+ "learning_rate": 2.788402433740517e-06,
3326
+ "loss": 0.3973,
3327
+ "step": 23700
3328
+ },
3329
+ {
3330
+ "epoch": 1.7901560262304967,
3331
+ "grad_norm": 0.17791134119033813,
3332
+ "learning_rate": 2.691764107389977e-06,
3333
+ "loss": 0.3959,
3334
+ "step": 23750
3335
+ },
3336
+ {
3337
+ "epoch": 1.793924775759403,
3338
+ "grad_norm": 0.1789197474718094,
3339
+ "learning_rate": 2.596783724781282e-06,
3340
+ "loss": 0.3974,
3341
+ "step": 23800
3342
+ },
3343
+ {
3344
+ "epoch": 1.7976935252883095,
3345
+ "grad_norm": 0.17759652435779572,
3346
+ "learning_rate": 2.5034646145553788e-06,
3347
+ "loss": 0.3974,
3348
+ "step": 23850
3349
+ },
3350
+ {
3351
+ "epoch": 1.8014622748172155,
3352
+ "grad_norm": 0.1805141419172287,
3353
+ "learning_rate": 2.4118100471329787e-06,
3354
+ "loss": 0.3959,
3355
+ "step": 23900
3356
+ },
3357
+ {
3358
+ "epoch": 1.805231024346122,
3359
+ "grad_norm": 0.18057376146316528,
3360
+ "learning_rate": 2.3218232345999625e-06,
3361
+ "loss": 0.3969,
3362
+ "step": 23950
3363
+ },
3364
+ {
3365
+ "epoch": 1.8089997738750283,
3366
+ "grad_norm": 0.1702955961227417,
3367
+ "learning_rate": 2.2335073305948086e-06,
3368
+ "loss": 0.394,
3369
+ "step": 24000
3370
+ },
3371
+ {
3372
+ "epoch": 1.8127685234039346,
3373
+ "grad_norm": 0.1743067502975464,
3374
+ "learning_rate": 2.1468654301980527e-06,
3375
+ "loss": 0.397,
3376
+ "step": 24050
3377
+ },
3378
+ {
3379
+ "epoch": 1.8165372729328408,
3380
+ "grad_norm": 0.17483128607273102,
3381
+ "learning_rate": 2.0619005698238437e-06,
3382
+ "loss": 0.3975,
3383
+ "step": 24100
3384
+ },
3385
+ {
3386
+ "epoch": 1.820306022461747,
3387
+ "grad_norm": 0.18305592238903046,
3388
+ "learning_rate": 1.9786157271135032e-06,
3389
+ "loss": 0.3971,
3390
+ "step": 24150
3391
+ },
3392
+ {
3393
+ "epoch": 1.8240747719906536,
3394
+ "grad_norm": 0.17903833091259003,
3395
+ "learning_rate": 1.8970138208311949e-06,
3396
+ "loss": 0.3973,
3397
+ "step": 24200
3398
+ },
3399
+ {
3400
+ "epoch": 1.8278435215195596,
3401
+ "grad_norm": 0.18153904378414154,
3402
+ "learning_rate": 1.817097710761656e-06,
3403
+ "loss": 0.3944,
3404
+ "step": 24250
3405
+ },
3406
+ {
3407
+ "epoch": 1.8316122710484661,
3408
+ "grad_norm": 0.1754598170518875,
3409
+ "learning_rate": 1.7388701976099041e-06,
3410
+ "loss": 0.3953,
3411
+ "step": 24300
3412
+ },
3413
+ {
3414
+ "epoch": 1.8353810205773724,
3415
+ "grad_norm": 0.18127521872520447,
3416
+ "learning_rate": 1.6623340229031503e-06,
3417
+ "loss": 0.3973,
3418
+ "step": 24350
3419
+ },
3420
+ {
3421
+ "epoch": 1.8391497701062787,
3422
+ "grad_norm": 0.17995725572109222,
3423
+ "learning_rate": 1.5874918688946972e-06,
3424
+ "loss": 0.3975,
3425
+ "step": 24400
3426
+ },
3427
+ {
3428
+ "epoch": 1.8429185196351852,
3429
+ "grad_norm": 0.18092754483222961,
3430
+ "learning_rate": 1.5143463584699424e-06,
3431
+ "loss": 0.3955,
3432
+ "step": 24450
3433
+ },
3434
+ {
3435
+ "epoch": 1.8466872691640912,
3436
+ "grad_norm": 0.1810247302055359,
3437
+ "learning_rate": 1.4429000550544414e-06,
3438
+ "loss": 0.3948,
3439
+ "step": 24500
3440
+ },
3441
+ {
3442
+ "epoch": 1.8504560186929977,
3443
+ "grad_norm": 0.18044689297676086,
3444
+ "learning_rate": 1.3731554625241006e-06,
3445
+ "loss": 0.3974,
3446
+ "step": 24550
3447
+ },
3448
+ {
3449
+ "epoch": 1.854224768221904,
3450
+ "grad_norm": 0.17675849795341492,
3451
+ "learning_rate": 1.305115025117387e-06,
3452
+ "loss": 0.3968,
3453
+ "step": 24600
3454
+ },
3455
+ {
3456
+ "epoch": 1.8579935177508102,
3457
+ "grad_norm": 0.17403309047222137,
3458
+ "learning_rate": 1.2387811273497252e-06,
3459
+ "loss": 0.3973,
3460
+ "step": 24650
3461
+ },
3462
+ {
3463
+ "epoch": 1.8617622672797167,
3464
+ "grad_norm": 0.17401446402072906,
3465
+ "learning_rate": 1.1741560939298791e-06,
3466
+ "loss": 0.3956,
3467
+ "step": 24700
3468
+ },
3469
+ {
3470
+ "epoch": 1.8655310168086228,
3471
+ "grad_norm": 0.18209025263786316,
3472
+ "learning_rate": 1.1112421896784853e-06,
3473
+ "loss": 0.3959,
3474
+ "step": 24750
3475
+ },
3476
+ {
3477
+ "epoch": 1.8692997663375293,
3478
+ "grad_norm": 0.17801640927791595,
3479
+ "learning_rate": 1.0500416194487384e-06,
3480
+ "loss": 0.3966,
3481
+ "step": 24800
3482
+ },
3483
+ {
3484
+ "epoch": 1.8730685158664355,
3485
+ "grad_norm": 0.17677666246891022,
3486
+ "learning_rate": 9.905565280490302e-07,
3487
+ "loss": 0.3983,
3488
+ "step": 24850
3489
+ },
3490
+ {
3491
+ "epoch": 1.8768372653953418,
3492
+ "grad_norm": 0.17637313902378082,
3493
+ "learning_rate": 9.327890001678719e-07,
3494
+ "loss": 0.3974,
3495
+ "step": 24900
3496
+ },
3497
+ {
3498
+ "epoch": 1.8806060149242483,
3499
+ "grad_norm": 0.17628493905067444,
3500
+ "learning_rate": 8.76741060300762e-07,
3501
+ "loss": 0.3967,
3502
+ "step": 24950
3503
+ },
3504
+ {
3505
+ "epoch": 1.8843747644531543,
3506
+ "grad_norm": 0.17734892666339874,
3507
+ "learning_rate": 8.224146726792947e-07,
3508
+ "loss": 0.3936,
3509
+ "step": 25000
3510
+ },
3511
+ {
3512
+ "epoch": 1.8881435139820608,
3513
+ "grad_norm": 0.17440645396709442,
3514
+ "learning_rate": 7.698117412022864e-07,
3515
+ "loss": 0.3964,
3516
+ "step": 25050
3517
+ },
3518
+ {
3519
+ "epoch": 1.891912263510967,
3520
+ "grad_norm": 0.1781737059354782,
3521
+ "learning_rate": 7.189341093690627e-07,
3522
+ "loss": 0.3961,
3523
+ "step": 25100
3524
+ },
3525
+ {
3526
+ "epoch": 1.8956810130398734,
3527
+ "grad_norm": 0.17803654074668884,
3528
+ "learning_rate": 6.697835602148483e-07,
3529
+ "loss": 0.3943,
3530
+ "step": 25150
3531
+ },
3532
+ {
3533
+ "epoch": 1.8994497625687798,
3534
+ "grad_norm": 0.17455802857875824,
3535
+ "learning_rate": 6.223618162483014e-07,
3536
+ "loss": 0.3958,
3537
+ "step": 25200
3538
+ },
3539
+ {
3540
+ "epoch": 1.9032185120976859,
3541
+ "grad_norm": 0.17685271799564362,
3542
+ "learning_rate": 5.766705393911165e-07,
3543
+ "loss": 0.3968,
3544
+ "step": 25250
3545
+ },
3546
+ {
3547
+ "epoch": 1.9069872616265924,
3548
+ "grad_norm": 0.18048785626888275,
3549
+ "learning_rate": 5.327113309197828e-07,
3550
+ "loss": 0.3944,
3551
+ "step": 25300
3552
+ },
3553
+ {
3554
+ "epoch": 1.9107560111554986,
3555
+ "grad_norm": 0.17648503184318542,
3556
+ "learning_rate": 4.904857314095068e-07,
3557
+ "loss": 0.3954,
3558
+ "step": 25350
3559
+ },
3560
+ {
3561
+ "epoch": 1.914524760684405,
3562
+ "grad_norm": 0.17615610361099243,
3563
+ "learning_rate": 4.4999522068017164e-07,
3564
+ "loss": 0.3963,
3565
+ "step": 25400
3566
+ },
3567
+ {
3568
+ "epoch": 1.9182935102133112,
3569
+ "grad_norm": 0.17669633030891418,
3570
+ "learning_rate": 4.112412177445124e-07,
3571
+ "loss": 0.3965,
3572
+ "step": 25450
3573
+ },
3574
+ {
3575
+ "epoch": 1.9220622597422174,
3576
+ "grad_norm": 0.17653539776802063,
3577
+ "learning_rate": 3.7422508075835583e-07,
3578
+ "loss": 0.397,
3579
+ "step": 25500
3580
+ },
3581
+ {
3582
+ "epoch": 1.925831009271124,
3583
+ "grad_norm": 0.17871299386024475,
3584
+ "learning_rate": 3.3894810697305267e-07,
3585
+ "loss": 0.3952,
3586
+ "step": 25550
3587
+ },
3588
+ {
3589
+ "epoch": 1.92959975880003,
3590
+ "grad_norm": 0.18262889981269836,
3591
+ "learning_rate": 3.05411532689992e-07,
3592
+ "loss": 0.3979,
3593
+ "step": 25600
3594
+ },
3595
+ {
3596
+ "epoch": 1.9333685083289365,
3597
+ "grad_norm": 0.1723639965057373,
3598
+ "learning_rate": 2.7361653321729133e-07,
3599
+ "loss": 0.3959,
3600
+ "step": 25650
3601
+ },
3602
+ {
3603
+ "epoch": 1.9371372578578427,
3604
+ "grad_norm": 0.1799677610397339,
3605
+ "learning_rate": 2.435642228285906e-07,
3606
+ "loss": 0.3959,
3607
+ "step": 25700
3608
+ },
3609
+ {
3610
+ "epoch": 1.940906007386749,
3611
+ "grad_norm": 0.18162404000759125,
3612
+ "learning_rate": 2.15255654724017e-07,
3613
+ "loss": 0.3957,
3614
+ "step": 25750
3615
+ },
3616
+ {
3617
+ "epoch": 1.9446747569156555,
3618
+ "grad_norm": 0.17320673167705536,
3619
+ "learning_rate": 1.886918209932642e-07,
3620
+ "loss": 0.395,
3621
+ "step": 25800
3622
+ },
3623
+ {
3624
+ "epoch": 1.9484435064445615,
3625
+ "grad_norm": 0.1870715469121933,
3626
+ "learning_rate": 1.63873652580826e-07,
3627
+ "loss": 0.3966,
3628
+ "step": 25850
3629
+ },
3630
+ {
3631
+ "epoch": 1.952212255973468,
3632
+ "grad_norm": 0.1799531877040863,
3633
+ "learning_rate": 1.4080201925338322e-07,
3634
+ "loss": 0.397,
3635
+ "step": 25900
3636
+ },
3637
+ {
3638
+ "epoch": 1.9559810055023743,
3639
+ "grad_norm": 0.18043088912963867,
3640
+ "learning_rate": 1.1947772956930615e-07,
3641
+ "loss": 0.3956,
3642
+ "step": 25950
3643
+ },
3644
+ {
3645
+ "epoch": 1.9597497550312806,
3646
+ "grad_norm": 0.18167664110660553,
3647
+ "learning_rate": 9.99015308503215e-08,
3648
+ "loss": 0.3967,
3649
+ "step": 26000
3650
+ },
3651
+ {
3652
+ "epoch": 1.963518504560187,
3653
+ "grad_norm": 0.1858936846256256,
3654
+ "learning_rate": 8.207410915532232e-08,
3655
+ "loss": 0.3968,
3656
+ "step": 26050
3657
+ },
3658
+ {
3659
+ "epoch": 1.967287254089093,
3660
+ "grad_norm": 0.17515628039836884,
3661
+ "learning_rate": 6.599608925633715e-08,
3662
+ "loss": 0.3951,
3663
+ "step": 26100
3664
+ },
3665
+ {
3666
+ "epoch": 1.9710560036179996,
3667
+ "grad_norm": 0.17847982048988342,
3668
+ "learning_rate": 5.166803461661429e-08,
3669
+ "loss": 0.3944,
3670
+ "step": 26150
3671
+ },
3672
+ {
3673
+ "epoch": 1.9748247531469059,
3674
+ "grad_norm": 0.17742793262004852,
3675
+ "learning_rate": 3.909044737089307e-08,
3676
+ "loss": 0.3963,
3677
+ "step": 26200
3678
+ },
3679
+ {
3680
+ "epoch": 1.9785935026758121,
3681
+ "grad_norm": 0.17997916042804718,
3682
+ "learning_rate": 2.826376830779576e-08,
3683
+ "loss": 0.3948,
3684
+ "step": 26250
3685
+ },
3686
+ {
3687
+ "epoch": 1.9823622522047186,
3688
+ "grad_norm": 0.1779303252696991,
3689
+ "learning_rate": 1.9188376854373246e-08,
3690
+ "loss": 0.396,
3691
+ "step": 26300
3692
+ },
3693
+ {
3694
+ "epoch": 1.9861310017336247,
3695
+ "grad_norm": 0.18305301666259766,
3696
+ "learning_rate": 1.1864591062832331e-08,
3697
+ "loss": 0.397,
3698
+ "step": 26350
3699
+ },
3700
+ {
3701
+ "epoch": 1.9898997512625312,
3702
+ "grad_norm": 0.1764865517616272,
3703
+ "learning_rate": 6.292667599366864e-09,
3704
+ "loss": 0.3953,
3705
+ "step": 26400
3706
+ },
3707
+ {
3708
+ "epoch": 1.9936685007914374,
3709
+ "grad_norm": 0.17203836143016815,
3710
+ "learning_rate": 2.4728017351649534e-09,
3711
+ "loss": 0.3968,
3712
+ "step": 26450
3713
+ },
3714
+ {
3715
+ "epoch": 1.9974372503203437,
3716
+ "grad_norm": 0.17117449641227722,
3717
+ "learning_rate": 4.0512733956998837e-10,
3718
+ "loss": 0.3977,
3719
+ "step": 26500
3720
+ }
3721
+ ],
3722
+ "logging_steps": 50,
3723
+ "max_steps": 26534,
3724
+ "num_input_tokens_seen": 0,
3725
+ "num_train_epochs": 2,
3726
+ "save_steps": 500,
3727
+ "stateful_callbacks": {
3728
+ "TrainerControl": {
3729
+ "args": {
3730
+ "should_epoch_stop": false,
3731
+ "should_evaluate": false,
3732
+ "should_log": false,
3733
+ "should_save": true,
3734
+ "should_training_stop": true
3735
+ },
3736
+ "attributes": {}
3737
+ }
3738
+ },
3739
+ "total_flos": 1.017526223018121e+20,
3740
+ "train_batch_size": 2,
3741
+ "trial_name": null,
3742
+ "trial_params": null
3743
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55a5d4f5abe64380c48a5df9e1f9c993ab2e012dc04d83f42a28f683502f1a0f
3
+ size 7288
zero_to_fp32.py ADDED
@@ -0,0 +1,674 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example:
14
+ # python zero_to_fp32.py . output_dir/
15
+ # or
16
+ # python zero_to_fp32.py . output_dir/ --safe_serialization
17
+
18
+ import argparse
19
+ import torch
20
+ import glob
21
+ import math
22
+ import os
23
+ import re
24
+ import json
25
+ from tqdm import tqdm
26
+ from collections import OrderedDict
27
+ from dataclasses import dataclass
28
+
29
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
30
+ # DeepSpeed data structures it has to be available in the current python environment.
31
+ from deepspeed.utils import logger
32
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
33
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
34
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
35
+
36
+
37
+ @dataclass
38
+ class zero_model_state:
39
+ buffers: dict()
40
+ param_shapes: dict()
41
+ shared_params: list
42
+ ds_version: int
43
+ frozen_param_shapes: dict()
44
+ frozen_param_fragments: dict()
45
+
46
+
47
+ debug = 0
48
+
49
+ # load to cpu
50
+ device = torch.device('cpu')
51
+
52
+
53
+ def atoi(text):
54
+ return int(text) if text.isdigit() else text
55
+
56
+
57
+ def natural_keys(text):
58
+ '''
59
+ alist.sort(key=natural_keys) sorts in human order
60
+ http://nedbatchelder.com/blog/200712/human_sorting.html
61
+ (See Toothy's implementation in the comments)
62
+ '''
63
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
64
+
65
+
66
+ def get_model_state_file(checkpoint_dir, zero_stage):
67
+ if not os.path.isdir(checkpoint_dir):
68
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
69
+
70
+ # there should be only one file
71
+ if zero_stage <= 2:
72
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
73
+ elif zero_stage == 3:
74
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
75
+
76
+ if not os.path.exists(file):
77
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
78
+
79
+ return file
80
+
81
+
82
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
83
+ # XXX: need to test that this simple glob rule works for multi-node setup too
84
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
85
+
86
+ if len(ckpt_files) == 0:
87
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
88
+
89
+ return ckpt_files
90
+
91
+
92
+ def get_optim_files(checkpoint_dir):
93
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
94
+
95
+
96
+ def get_model_state_files(checkpoint_dir):
97
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
98
+
99
+
100
+ def parse_model_states(files):
101
+ zero_model_states = []
102
+ for file in files:
103
+ state_dict = torch.load(file, map_location=device)
104
+
105
+ if BUFFER_NAMES not in state_dict:
106
+ raise ValueError(f"{file} is not a model state checkpoint")
107
+ buffer_names = state_dict[BUFFER_NAMES]
108
+ if debug:
109
+ print("Found buffers:", buffer_names)
110
+
111
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
112
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
113
+ param_shapes = state_dict[PARAM_SHAPES]
114
+
115
+ # collect parameters that are included in param_shapes
116
+ param_names = []
117
+ for s in param_shapes:
118
+ for name in s.keys():
119
+ param_names.append(name)
120
+
121
+ # update with frozen parameters
122
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
123
+ if frozen_param_shapes is not None:
124
+ if debug:
125
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
126
+ param_names += list(frozen_param_shapes.keys())
127
+
128
+ # handle shared params
129
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
130
+
131
+ ds_version = state_dict.get(DS_VERSION, None)
132
+
133
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
134
+
135
+ z_model_state = zero_model_state(buffers=buffers,
136
+ param_shapes=param_shapes,
137
+ shared_params=shared_params,
138
+ ds_version=ds_version,
139
+ frozen_param_shapes=frozen_param_shapes,
140
+ frozen_param_fragments=frozen_param_fragments)
141
+ zero_model_states.append(z_model_state)
142
+
143
+ return zero_model_states
144
+
145
+
146
+ def parse_optim_states(files, ds_checkpoint_dir):
147
+ total_files = len(files)
148
+ state_dicts = []
149
+ for f in files:
150
+ state_dict = torch.load(f, map_location=device)
151
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
152
+ # and also handle the case where it was already removed by another helper script
153
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
154
+ state_dicts.append(state_dict)
155
+
156
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
157
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
158
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
159
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
160
+
161
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
162
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
163
+ # use the max of the partition_count to get the dp world_size.
164
+
165
+ if type(world_size) is list:
166
+ world_size = max(world_size)
167
+
168
+ if world_size != total_files:
169
+ raise ValueError(
170
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
171
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
172
+ )
173
+
174
+ # the groups are named differently in each stage
175
+ if zero_stage <= 2:
176
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
177
+ elif zero_stage == 3:
178
+ fp32_groups_key = FP32_FLAT_GROUPS
179
+ else:
180
+ raise ValueError(f"unknown zero stage {zero_stage}")
181
+
182
+ if zero_stage <= 2:
183
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
184
+ elif zero_stage == 3:
185
+ # if there is more than one param group, there will be multiple flattened tensors - one
186
+ # flattened tensor per group - for simplicity merge them into a single tensor
187
+ #
188
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
189
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
190
+
191
+ fp32_flat_groups = [
192
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
193
+ ]
194
+
195
+ return zero_stage, world_size, fp32_flat_groups
196
+
197
+
198
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
199
+ """
200
+ Returns fp32 state_dict reconstructed from ds checkpoint
201
+
202
+ Args:
203
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
204
+
205
+ """
206
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
207
+
208
+ optim_files = get_optim_files(ds_checkpoint_dir)
209
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
210
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
211
+
212
+ model_files = get_model_state_files(ds_checkpoint_dir)
213
+
214
+ zero_model_states = parse_model_states(model_files)
215
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
216
+
217
+ if zero_stage <= 2:
218
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
219
+ exclude_frozen_parameters)
220
+ elif zero_stage == 3:
221
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
222
+ exclude_frozen_parameters)
223
+
224
+
225
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
226
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
227
+ return
228
+
229
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
230
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
231
+
232
+ if debug:
233
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
234
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
235
+
236
+ wanted_params = len(frozen_param_shapes)
237
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
238
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
239
+ print(f'Frozen params: Have {avail_numel} numels to process.')
240
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
241
+
242
+ total_params = 0
243
+ total_numel = 0
244
+ for name, shape in frozen_param_shapes.items():
245
+ total_params += 1
246
+ unpartitioned_numel = shape.numel()
247
+ total_numel += unpartitioned_numel
248
+
249
+ state_dict[name] = frozen_param_fragments[name]
250
+
251
+ if debug:
252
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
253
+
254
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
255
+
256
+
257
+ def _has_callable(obj, fn):
258
+ attr = getattr(obj, fn, None)
259
+ return callable(attr)
260
+
261
+
262
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
263
+ param_shapes = zero_model_states[0].param_shapes
264
+
265
+ # Reconstruction protocol:
266
+ #
267
+ # XXX: document this
268
+
269
+ if debug:
270
+ for i in range(world_size):
271
+ for j in range(len(fp32_flat_groups[0])):
272
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
273
+
274
+ # XXX: memory usage doubles here (zero2)
275
+ num_param_groups = len(fp32_flat_groups[0])
276
+ merged_single_partition_of_fp32_groups = []
277
+ for i in range(num_param_groups):
278
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
279
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
280
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
281
+ avail_numel = sum(
282
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
283
+
284
+ if debug:
285
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
286
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
287
+ # not asserting if there is a mismatch due to possible padding
288
+ print(f"Have {avail_numel} numels to process.")
289
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
290
+
291
+ # params
292
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
293
+ # out-of-core computing solution
294
+ total_numel = 0
295
+ total_params = 0
296
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
297
+ offset = 0
298
+ avail_numel = full_single_fp32_vector.numel()
299
+ for name, shape in shapes.items():
300
+
301
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
302
+ total_numel += unpartitioned_numel
303
+ total_params += 1
304
+
305
+ if debug:
306
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
307
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
308
+ offset += unpartitioned_numel
309
+
310
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
311
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
312
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
313
+ # live optimizer object, so we are checking that the numbers are within the right range
314
+ align_to = 2 * world_size
315
+
316
+ def zero2_align(x):
317
+ return align_to * math.ceil(x / align_to)
318
+
319
+ if debug:
320
+ print(f"original offset={offset}, avail_numel={avail_numel}")
321
+
322
+ offset = zero2_align(offset)
323
+ avail_numel = zero2_align(avail_numel)
324
+
325
+ if debug:
326
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
327
+
328
+ # Sanity check
329
+ if offset != avail_numel:
330
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
331
+
332
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
333
+
334
+
335
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
336
+ exclude_frozen_parameters):
337
+ state_dict = OrderedDict()
338
+
339
+ # buffers
340
+ buffers = zero_model_states[0].buffers
341
+ state_dict.update(buffers)
342
+ if debug:
343
+ print(f"added {len(buffers)} buffers")
344
+
345
+ if not exclude_frozen_parameters:
346
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
347
+
348
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
349
+
350
+ # recover shared parameters
351
+ for pair in zero_model_states[0].shared_params:
352
+ if pair[1] in state_dict:
353
+ state_dict[pair[0]] = state_dict[pair[1]]
354
+
355
+ return state_dict
356
+
357
+
358
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
359
+ remainder = unpartitioned_numel % world_size
360
+ padding_numel = (world_size - remainder) if remainder else 0
361
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
362
+ return partitioned_numel, padding_numel
363
+
364
+
365
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
366
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
367
+ return
368
+
369
+ if debug:
370
+ for i in range(world_size):
371
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
372
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
373
+
374
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
375
+ wanted_params = len(frozen_param_shapes)
376
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
377
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
378
+ print(f'Frozen params: Have {avail_numel} numels to process.')
379
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
380
+
381
+ total_params = 0
382
+ total_numel = 0
383
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
384
+ total_params += 1
385
+ unpartitioned_numel = shape.numel()
386
+ total_numel += unpartitioned_numel
387
+
388
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
389
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
390
+
391
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
392
+
393
+ if debug:
394
+ print(
395
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
396
+ )
397
+
398
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
399
+
400
+
401
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
402
+ param_shapes = zero_model_states[0].param_shapes
403
+ avail_numel = fp32_flat_groups[0].numel() * world_size
404
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
405
+ # param, re-consolidating each param, while dealing with padding if any
406
+
407
+ # merge list of dicts, preserving order
408
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
409
+
410
+ if debug:
411
+ for i in range(world_size):
412
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
413
+
414
+ wanted_params = len(param_shapes)
415
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
416
+ # not asserting if there is a mismatch due to possible padding
417
+ avail_numel = fp32_flat_groups[0].numel() * world_size
418
+ print(f"Trainable params: Have {avail_numel} numels to process.")
419
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
420
+
421
+ # params
422
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
423
+ # out-of-core computing solution
424
+ offset = 0
425
+ total_numel = 0
426
+ total_params = 0
427
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering Sharded Weights'):
428
+ unpartitioned_numel = shape.numel()
429
+ total_numel += unpartitioned_numel
430
+ total_params += 1
431
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
432
+
433
+ if debug:
434
+ print(
435
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
436
+ )
437
+
438
+ # XXX: memory usage doubles here
439
+ state_dict[name] = torch.cat(
440
+ tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
441
+ 0).narrow(0, 0, unpartitioned_numel).view(shape)
442
+ offset += partitioned_numel
443
+
444
+ offset *= world_size
445
+
446
+ # Sanity check
447
+ if offset != avail_numel:
448
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
449
+
450
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
451
+
452
+
453
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
454
+ exclude_frozen_parameters):
455
+ state_dict = OrderedDict()
456
+
457
+ # buffers
458
+ buffers = zero_model_states[0].buffers
459
+ state_dict.update(buffers)
460
+ if debug:
461
+ print(f"added {len(buffers)} buffers")
462
+
463
+ if not exclude_frozen_parameters:
464
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
465
+
466
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
467
+
468
+ # recover shared parameters
469
+ for pair in zero_model_states[0].shared_params:
470
+ if pair[1] in state_dict:
471
+ state_dict[pair[0]] = state_dict[pair[1]]
472
+
473
+ return state_dict
474
+
475
+
476
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
477
+ """
478
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
479
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
480
+ via a model hub.
481
+
482
+ Args:
483
+ - ``checkpoint_dir``: path to the desired checkpoint folder
484
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
485
+ - ``exclude_frozen_parameters``: exclude frozen parameters
486
+
487
+ Returns:
488
+ - pytorch ``state_dict``
489
+
490
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
491
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
492
+ the checkpoint.
493
+
494
+ A typical usage might be ::
495
+
496
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
497
+ # do the training and checkpoint saving
498
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
499
+ model = model.cpu() # move to cpu
500
+ model.load_state_dict(state_dict)
501
+ # submit to model hub or save the model to share with others
502
+
503
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
504
+ application. i.e. you will need to re-initialize the deepspeed engine, since
505
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
506
+
507
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
508
+
509
+ """
510
+ if tag is None:
511
+ latest_path = os.path.join(checkpoint_dir, 'latest')
512
+ if os.path.isfile(latest_path):
513
+ with open(latest_path, 'r') as fd:
514
+ tag = fd.read().strip()
515
+ else:
516
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
517
+
518
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
519
+
520
+ if not os.path.isdir(ds_checkpoint_dir):
521
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
522
+
523
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
524
+
525
+
526
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
527
+ output_dir,
528
+ max_shard_size="5GB",
529
+ safe_serialization=False,
530
+ tag=None,
531
+ exclude_frozen_parameters=False):
532
+ """
533
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
534
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
535
+
536
+ Args:
537
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
538
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
539
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
540
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
541
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
542
+ - ``exclude_frozen_parameters``: exclude frozen parameters
543
+ """
544
+ # Dependency pre-check
545
+ if safe_serialization:
546
+ try:
547
+ from safetensors.torch import save_file
548
+ except ImportError:
549
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
550
+ raise
551
+ if max_shard_size is not None:
552
+ try:
553
+ from huggingface_hub import split_torch_state_dict_into_shards
554
+ except ImportError:
555
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
556
+ raise
557
+
558
+ # Convert zero checkpoint to state_dict
559
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
560
+
561
+ # Shard the model if it is too big.
562
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
563
+ if max_shard_size is not None:
564
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
565
+ state_dict_split = split_torch_state_dict_into_shards(state_dict,
566
+ filename_pattern=filename_pattern,
567
+ max_shard_size=max_shard_size)
568
+ else:
569
+ from collections import namedtuple
570
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
571
+ state_dict_split = StateDictSplit(is_sharded=False,
572
+ filename_to_tensors={weights_name: list(state_dict.keys())})
573
+
574
+ # Save the model
575
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
576
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
577
+ shard = {tensor: state_dict[tensor].contiguous() for tensor in tensors}
578
+ output_path = os.path.join(output_dir, shard_file)
579
+ if safe_serialization:
580
+ save_file(shard, output_path, metadata={"format": "pt"})
581
+ else:
582
+ torch.save(shard, output_path)
583
+
584
+ # Save index if sharded
585
+ if state_dict_split.is_sharded:
586
+ index = {
587
+ "metadata": state_dict_split.metadata,
588
+ "weight_map": state_dict_split.tensor_to_filename,
589
+ }
590
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
591
+ save_index_file = os.path.join(output_dir, save_index_file)
592
+ with open(save_index_file, "w", encoding="utf-8") as f:
593
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
594
+ f.write(content)
595
+
596
+
597
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
598
+ """
599
+ 1. Put the provided model to cpu
600
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
601
+ 3. Load it into the provided model
602
+
603
+ Args:
604
+ - ``model``: the model object to update
605
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
606
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
607
+
608
+ Returns:
609
+ - ``model`: modified model
610
+
611
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
612
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
613
+ conveniently placed for you in the checkpoint folder.
614
+
615
+ A typical usage might be ::
616
+
617
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
618
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
619
+ # submit to model hub or save the model to share with others
620
+
621
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
622
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
623
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
624
+
625
+ """
626
+ logger.info(f"Extracting fp32 weights")
627
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
628
+
629
+ logger.info(f"Overwriting model with fp32 weights")
630
+ model = model.cpu()
631
+ model.load_state_dict(state_dict, strict=False)
632
+
633
+ return model
634
+
635
+
636
+ if __name__ == "__main__":
637
+ parser = argparse.ArgumentParser()
638
+ parser.add_argument("checkpoint_dir",
639
+ type=str,
640
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
641
+ parser.add_argument("output_dir",
642
+ type=str,
643
+ help="directory to the pytorch fp32 state_dict output files"
644
+ "(e.g. path/checkpoint-12-output/)")
645
+ parser.add_argument(
646
+ "--max_shard_size",
647
+ type=str,
648
+ default="5GB",
649
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
650
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
651
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
652
+ "without CPU OOM issues.")
653
+ parser.add_argument(
654
+ "--safe_serialization",
655
+ default=False,
656
+ action='store_true',
657
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
658
+ parser.add_argument("-t",
659
+ "--tag",
660
+ type=str,
661
+ default=None,
662
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
663
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
664
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
665
+ args = parser.parse_args()
666
+
667
+ debug = args.debug
668
+
669
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
670
+ args.output_dir,
671
+ max_shard_size=args.max_shard_size,
672
+ safe_serialization=args.safe_serialization,
673
+ tag=args.tag,
674
+ exclude_frozen_parameters=args.exclude_frozen_parameters)