yukidump commited on
Commit
dad50a2
·
1 Parent(s): 024481b
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/config.json ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlavaInterleaveForConditionalGeneration"
4
+ ],
5
+ "btadapter": true,
6
+ "btadapter_depth": 4,
7
+ "clip_post_pretrain": null,
8
+ "clip_weight": "/group/40043/ruyangliu/weight/CLIP-L334",
9
+ "frame_shape": [
10
+ 24,
11
+ 24
12
+ ],
13
+ "hidden_size": 4096,
14
+ "ignore_index": -100,
15
+ "image_grid_pinpoints": [
16
+ [
17
+ 336,
18
+ 672
19
+ ],
20
+ [
21
+ 672,
22
+ 336
23
+ ],
24
+ [
25
+ 672,
26
+ 672
27
+ ],
28
+ [
29
+ 1008,
30
+ 336
31
+ ],
32
+ [
33
+ 336,
34
+ 1008
35
+ ]
36
+ ],
37
+ "image_pooling_kernel": [
38
+ 1,
39
+ 3,
40
+ 3
41
+ ],
42
+ "image_pooling_stride": [
43
+ 1,
44
+ 3,
45
+ 3
46
+ ],
47
+ "image_token_index": 32000,
48
+ "long_clip": true,
49
+ "max_T": 64,
50
+ "model_type": "llava_next",
51
+ "pad_token_id": 0,
52
+ "pllava_pooling_shape": null,
53
+ "pooling": "clipST_3d",
54
+ "pooling_kernel": [
55
+ 2,
56
+ 3,
57
+ 3
58
+ ],
59
+ "pooling_stride": [
60
+ 2,
61
+ 3,
62
+ 3
63
+ ],
64
+ "pooling_temp": 0.01,
65
+ "projector_hidden_act": "gelu",
66
+ "text_config": {
67
+ "_name_or_path": "lmsys/vicuna-7b-v1.5",
68
+ "architectures": [
69
+ "LlamaForCausalLM"
70
+ ],
71
+ "max_position_embeddings": 4096,
72
+ "model_type": "llama",
73
+ "pad_token_id": 0,
74
+ "rms_norm_eps": 1e-05,
75
+ "torch_dtype": "float16",
76
+ "vocab_size": 32064
77
+ },
78
+ "torch_dtype": "bfloat16",
79
+ "transformers_version": "4.40.0",
80
+ "use_image_newline_parameter": true,
81
+ "vision_config": {
82
+ "depth": 4,
83
+ "hidden_size": 1024,
84
+ "image_size": 336,
85
+ "intermediate_size": 4096,
86
+ "max_T": 64,
87
+ "model_type": "clip_vision_model",
88
+ "num_attention_heads": 16,
89
+ "num_hidden_layers": 24,
90
+ "patch_size": 14,
91
+ "projection_dim": 768,
92
+ "vocab_size": 32000
93
+ },
94
+ "vision_feature_layer": -2,
95
+ "vision_feature_select_strategy": "default",
96
+ "vocab_size": 32064
97
+ }
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.40.0"
7
+ }
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/model-00001-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:badbca0f8b6f276fcf9f0b6f35ad29370715f5d9e5bde69804ab78457e611ced
3
+ size 4980641554
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/model-00002-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c486b79a3c76b9a43cfd068b592bfc88cfac2e5564d2552c55b24b518581eb0d
3
+ size 4957878536
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/model-00003-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4df79bed0e8f9f55e5457c7bc44022a71d5a4d7f5e4402b4da25a356401f2cf
3
+ size 4580905400
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/trainer_state.json ADDED
@@ -0,0 +1,422 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9999115122555526,
5
+ "eval_steps": 500,
6
+ "global_step": 2825,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.017697548889478807,
13
+ "grad_norm": 1.3246452072458292,
14
+ "learning_rate": 1.1764705882352942e-05,
15
+ "loss": 1.3482,
16
+ "step": 50
17
+ },
18
+ {
19
+ "epoch": 0.035395097778957614,
20
+ "grad_norm": 1.184986868355016,
21
+ "learning_rate": 1.9998521094455198e-05,
22
+ "loss": 1.0655,
23
+ "step": 100
24
+ },
25
+ {
26
+ "epoch": 0.053092646668436425,
27
+ "grad_norm": 0.9956602761532123,
28
+ "learning_rate": 1.9972241607451552e-05,
29
+ "loss": 1.0325,
30
+ "step": 150
31
+ },
32
+ {
33
+ "epoch": 0.07079019555791523,
34
+ "grad_norm": 0.8985264278021503,
35
+ "learning_rate": 1.9913196946839304e-05,
36
+ "loss": 1.0115,
37
+ "step": 200
38
+ },
39
+ {
40
+ "epoch": 0.08848774444739403,
41
+ "grad_norm": 0.9761536009178263,
42
+ "learning_rate": 1.9821581111985072e-05,
43
+ "loss": 1.0087,
44
+ "step": 250
45
+ },
46
+ {
47
+ "epoch": 0.10618529333687285,
48
+ "grad_norm": 0.9582085355410745,
49
+ "learning_rate": 1.9697695119335547e-05,
50
+ "loss": 0.9959,
51
+ "step": 300
52
+ },
53
+ {
54
+ "epoch": 0.12388284222635165,
55
+ "grad_norm": 0.9502507945345694,
56
+ "learning_rate": 1.954194601338651e-05,
57
+ "loss": 0.9832,
58
+ "step": 350
59
+ },
60
+ {
61
+ "epoch": 0.14158039111583046,
62
+ "grad_norm": 0.9312008953589136,
63
+ "learning_rate": 1.9354845529282042e-05,
64
+ "loss": 0.9838,
65
+ "step": 400
66
+ },
67
+ {
68
+ "epoch": 0.15927794000530926,
69
+ "grad_norm": 0.9921814925845447,
70
+ "learning_rate": 1.9137008411438213e-05,
71
+ "loss": 0.9743,
72
+ "step": 450
73
+ },
74
+ {
75
+ "epoch": 0.17697548889478806,
76
+ "grad_norm": 0.9946357977754318,
77
+ "learning_rate": 1.8889150393715627e-05,
78
+ "loss": 0.9674,
79
+ "step": 500
80
+ },
81
+ {
82
+ "epoch": 0.19467303778426687,
83
+ "grad_norm": 0.936001151695521,
84
+ "learning_rate": 1.8612085847777215e-05,
85
+ "loss": 0.9679,
86
+ "step": 550
87
+ },
88
+ {
89
+ "epoch": 0.2123705866737457,
90
+ "grad_norm": 0.9739097270294552,
91
+ "learning_rate": 1.8306725107357933e-05,
92
+ "loss": 0.9622,
93
+ "step": 600
94
+ },
95
+ {
96
+ "epoch": 0.2300681355632245,
97
+ "grad_norm": 1.0799028416482208,
98
+ "learning_rate": 1.7974071477237887e-05,
99
+ "loss": 0.976,
100
+ "step": 650
101
+ },
102
+ {
103
+ "epoch": 0.2477656844527033,
104
+ "grad_norm": 0.9330950566244366,
105
+ "learning_rate": 1.7615217936746246e-05,
106
+ "loss": 0.9567,
107
+ "step": 700
108
+ },
109
+ {
110
+ "epoch": 0.2654632333421821,
111
+ "grad_norm": 0.8832817518420761,
112
+ "learning_rate": 1.7231343548627085e-05,
113
+ "loss": 0.9523,
114
+ "step": 750
115
+ },
116
+ {
117
+ "epoch": 0.2831607822316609,
118
+ "grad_norm": 0.9002124999100791,
119
+ "learning_rate": 1.6823709585066308e-05,
120
+ "loss": 0.9466,
121
+ "step": 800
122
+ },
123
+ {
124
+ "epoch": 0.30085833112113974,
125
+ "grad_norm": 0.9157647153217237,
126
+ "learning_rate": 1.6393655383608132e-05,
127
+ "loss": 0.9429,
128
+ "step": 850
129
+ },
130
+ {
131
+ "epoch": 0.3185558800106185,
132
+ "grad_norm": 5.464496113406752,
133
+ "learning_rate": 1.594259394657707e-05,
134
+ "loss": 0.9695,
135
+ "step": 900
136
+ },
137
+ {
138
+ "epoch": 0.33625342890009735,
139
+ "grad_norm": 0.9883051330334294,
140
+ "learning_rate": 1.5472007298464117e-05,
141
+ "loss": 0.944,
142
+ "step": 950
143
+ },
144
+ {
145
+ "epoch": 0.3539509777895761,
146
+ "grad_norm": 0.8832131211036971,
147
+ "learning_rate": 1.4983441616531152e-05,
148
+ "loss": 0.937,
149
+ "step": 1000
150
+ },
151
+ {
152
+ "epoch": 0.37164852667905496,
153
+ "grad_norm": 0.9595043070379917,
154
+ "learning_rate": 1.4478502150632503e-05,
155
+ "loss": 0.9313,
156
+ "step": 1050
157
+ },
158
+ {
159
+ "epoch": 0.38934607556853373,
160
+ "grad_norm": 1.035782879874017,
161
+ "learning_rate": 1.3958847948945428e-05,
162
+ "loss": 0.9264,
163
+ "step": 1100
164
+ },
165
+ {
166
+ "epoch": 0.40704362445801257,
167
+ "grad_norm": 0.961077579119761,
168
+ "learning_rate": 1.3426186406938769e-05,
169
+ "loss": 0.924,
170
+ "step": 1150
171
+ },
172
+ {
173
+ "epoch": 0.4247411733474914,
174
+ "grad_norm": 0.882487785914029,
175
+ "learning_rate": 1.2882267657489908e-05,
176
+ "loss": 0.9226,
177
+ "step": 1200
178
+ },
179
+ {
180
+ "epoch": 0.44243872223697017,
181
+ "grad_norm": 0.8676230908883836,
182
+ "learning_rate": 1.2328878820582122e-05,
183
+ "loss": 0.9134,
184
+ "step": 1250
185
+ },
186
+ {
187
+ "epoch": 0.460136271126449,
188
+ "grad_norm": 0.8663581800411811,
189
+ "learning_rate": 1.1767838131475654e-05,
190
+ "loss": 0.917,
191
+ "step": 1300
192
+ },
193
+ {
194
+ "epoch": 0.4778338200159278,
195
+ "grad_norm": 1.1913367079153512,
196
+ "learning_rate": 1.1200988966645286e-05,
197
+ "loss": 0.9181,
198
+ "step": 1350
199
+ },
200
+ {
201
+ "epoch": 0.4955313689054066,
202
+ "grad_norm": 0.8563122357771151,
203
+ "learning_rate": 1.0630193787112994e-05,
204
+ "loss": 0.903,
205
+ "step": 1400
206
+ },
207
+ {
208
+ "epoch": 0.5132289177948854,
209
+ "grad_norm": 0.8518715044091224,
210
+ "learning_rate": 1.005732801907567e-05,
211
+ "loss": 0.9085,
212
+ "step": 1450
213
+ },
214
+ {
215
+ "epoch": 0.5309264666843642,
216
+ "grad_norm": 0.9224776221871871,
217
+ "learning_rate": 9.484273891933982e-06,
218
+ "loss": 0.9086,
219
+ "step": 1500
220
+ },
221
+ {
222
+ "epoch": 0.548624015573843,
223
+ "grad_norm": 0.9846231662432429,
224
+ "learning_rate": 8.912914253968391e-06,
225
+ "loss": 0.9017,
226
+ "step": 1550
227
+ },
228
+ {
229
+ "epoch": 0.5663215644633218,
230
+ "grad_norm": 0.8883898612449065,
231
+ "learning_rate": 8.345126385981737e-06,
232
+ "loss": 0.896,
233
+ "step": 1600
234
+ },
235
+ {
236
+ "epoch": 0.5840191133528007,
237
+ "grad_norm": 0.9045281001313086,
238
+ "learning_rate": 7.782775833234522e-06,
239
+ "loss": 0.9033,
240
+ "step": 1650
241
+ },
242
+ {
243
+ "epoch": 0.6017166622422795,
244
+ "grad_norm": 0.8962005631393778,
245
+ "learning_rate": 7.227710275938987e-06,
246
+ "loss": 0.8979,
247
+ "step": 1700
248
+ },
249
+ {
250
+ "epoch": 0.6194142111317582,
251
+ "grad_norm": 0.8638732399362996,
252
+ "learning_rate": 6.68175345845119e-06,
253
+ "loss": 0.8889,
254
+ "step": 1750
255
+ },
256
+ {
257
+ "epoch": 0.637111760021237,
258
+ "grad_norm": 1.0900143725646285,
259
+ "learning_rate": 6.146699197107715e-06,
260
+ "loss": 0.8957,
261
+ "step": 1800
262
+ },
263
+ {
264
+ "epoch": 0.6548093089107159,
265
+ "grad_norm": 0.9103789814206331,
266
+ "learning_rate": 5.6243054863949675e-06,
267
+ "loss": 0.8858,
268
+ "step": 1850
269
+ },
270
+ {
271
+ "epoch": 0.6725068578001947,
272
+ "grad_norm": 0.9300189349082482,
273
+ "learning_rate": 5.116288722816087e-06,
274
+ "loss": 0.8822,
275
+ "step": 1900
276
+ },
277
+ {
278
+ "epoch": 0.6902044066896735,
279
+ "grad_norm": 0.8743852772084869,
280
+ "learning_rate": 4.6243180654337975e-06,
281
+ "loss": 0.9172,
282
+ "step": 1950
283
+ },
284
+ {
285
+ "epoch": 0.7079019555791523,
286
+ "grad_norm": 0.9546014887497036,
287
+ "learning_rate": 4.1500099516183555e-06,
288
+ "loss": 1.1497,
289
+ "step": 2000
290
+ },
291
+ {
292
+ "epoch": 0.7255995044686311,
293
+ "grad_norm": 0.9614448138630033,
294
+ "learning_rate": 3.6949227860198712e-06,
295
+ "loss": 1.0692,
296
+ "step": 2050
297
+ },
298
+ {
299
+ "epoch": 0.7432970533581099,
300
+ "grad_norm": 0.848345849582495,
301
+ "learning_rate": 3.2605518202151577e-06,
302
+ "loss": 1.0286,
303
+ "step": 2100
304
+ },
305
+ {
306
+ "epoch": 0.7609946022475887,
307
+ "grad_norm": 1.0998684999586237,
308
+ "learning_rate": 2.8483242398526723e-06,
309
+ "loss": 0.8803,
310
+ "step": 2150
311
+ },
312
+ {
313
+ "epoch": 0.7786921511370675,
314
+ "grad_norm": 0.8279471296348094,
315
+ "learning_rate": 2.4595944754374723e-06,
316
+ "loss": 0.8885,
317
+ "step": 2200
318
+ },
319
+ {
320
+ "epoch": 0.7963897000265463,
321
+ "grad_norm": 0.8602056476985009,
322
+ "learning_rate": 2.0956397521631666e-06,
323
+ "loss": 0.8704,
324
+ "step": 2250
325
+ },
326
+ {
327
+ "epoch": 0.8140872489160251,
328
+ "grad_norm": 0.9945305020433657,
329
+ "learning_rate": 1.757655893412622e-06,
330
+ "loss": 0.8677,
331
+ "step": 2300
332
+ },
333
+ {
334
+ "epoch": 0.831784797805504,
335
+ "grad_norm": 0.850430777727199,
336
+ "learning_rate": 1.4467533917154842e-06,
337
+ "loss": 0.877,
338
+ "step": 2350
339
+ },
340
+ {
341
+ "epoch": 0.8494823466949828,
342
+ "grad_norm": 0.896244091668865,
343
+ "learning_rate": 1.1639537600719764e-06,
344
+ "loss": 0.8752,
345
+ "step": 2400
346
+ },
347
+ {
348
+ "epoch": 0.8671798955844615,
349
+ "grad_norm": 0.9590853387394762,
350
+ "learning_rate": 9.101861756312369e-07,
351
+ "loss": 0.8737,
352
+ "step": 2450
353
+ },
354
+ {
355
+ "epoch": 0.8848774444739403,
356
+ "grad_norm": 0.805579311652691,
357
+ "learning_rate": 6.862844267517643e-07,
358
+ "loss": 0.8792,
359
+ "step": 2500
360
+ },
361
+ {
362
+ "epoch": 0.9025749933634192,
363
+ "grad_norm": 0.8767296521208048,
364
+ "learning_rate": 4.929841734749063e-07,
365
+ "loss": 0.875,
366
+ "step": 2550
367
+ },
368
+ {
369
+ "epoch": 0.920272542252898,
370
+ "grad_norm": 0.9004753221903812,
371
+ "learning_rate": 3.309205304124552e-07,
372
+ "loss": 0.8703,
373
+ "step": 2600
374
+ },
375
+ {
376
+ "epoch": 0.9379700911423767,
377
+ "grad_norm": 0.8622876486385493,
378
+ "learning_rate": 2.0062597999009114e-07,
379
+ "loss": 0.8815,
380
+ "step": 2650
381
+ },
382
+ {
383
+ "epoch": 0.9556676400318556,
384
+ "grad_norm": 0.896417353499626,
385
+ "learning_rate": 1.0252862290301092e-07,
386
+ "loss": 0.8715,
387
+ "step": 2700
388
+ },
389
+ {
390
+ "epoch": 0.9733651889213344,
391
+ "grad_norm": 0.9142726428404561,
392
+ "learning_rate": 3.6950771532126004e-08,
393
+ "loss": 0.8657,
394
+ "step": 2750
395
+ },
396
+ {
397
+ "epoch": 0.9910627378108132,
398
+ "grad_norm": 0.8664983124317569,
399
+ "learning_rate": 4.1078909423253325e-09,
400
+ "loss": 0.8791,
401
+ "step": 2800
402
+ },
403
+ {
404
+ "epoch": 0.9999115122555526,
405
+ "step": 2825,
406
+ "total_flos": 4.909801179132723e+16,
407
+ "train_loss": 0.9400995777771536,
408
+ "train_runtime": 88050.3139,
409
+ "train_samples_per_second": 0.513,
410
+ "train_steps_per_second": 0.032
411
+ }
412
+ ],
413
+ "logging_steps": 50,
414
+ "max_steps": 2825,
415
+ "num_input_tokens_seen": 0,
416
+ "num_train_epochs": 1,
417
+ "save_steps": 500,
418
+ "total_flos": 4.909801179132723e+16,
419
+ "train_batch_size": 4,
420
+ "trial_name": null,
421
+ "trial_params": null
422
+ }
llavav16bta_cp3d1688_longb_qafull_llhound_fullimg_multiimg_trainllm_e1_bz512_32f1688/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c61e1b93e453464c82a151998343e20e07c872d423dcc4a4a2e46bd6e53948b
3
+ size 7096