diff --git "a/ndarray-cache.json" "b/ndarray-cache.json" new file mode 100644--- /dev/null +++ "b/ndarray-cache.json" @@ -0,0 +1,3585 @@ +{ + "metadata": { + "ParamSize": 311, + "ParamBytes": 868547584.0, + "BitsPerParam": 4.501079412165634 + }, + "records": [ + { + "dataPath": "params_shard_0.bin", + "format": "raw-shard", + "nbytes": 116686848, + "records": [ + { + "name": "model.embed_tokens.q_weight", + "shape": [ + 151936, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 116686848, + "byteOffset": 0 + } + ], + "md5sum": "b4a76eb8de546a997d730508ce42b23d" + }, + { + "dataPath": "params_shard_1.bin", + "format": "raw-shard", + "nbytes": 22330368, + "records": [ + { + "name": "model.embed_tokens.q_scale", + "shape": [ + 151936, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 14585856, + "byteOffset": 0 + }, + { + "name": "model.layers.0.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 14585856 + }, + { + "name": "model.layers.0.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 14588928 + }, + { + "name": "model.layers.0.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 21470208 + } + ], + "md5sum": "a96259867668a051285d4861b87c8e33" + }, + { + "dataPath": "params_shard_2.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.0.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.0.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.0.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.0.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.0.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.0.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.1.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.1.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.1.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "39e7b00aa1090e015eaca7ab38a8b5bc" + }, + { + "dataPath": "params_shard_3.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.1.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.1.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.1.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.1.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.1.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.1.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.10.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.10.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.10.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "4bb40c69ce5626fa7a2518cc2ec00aa5" + }, + { + "dataPath": "params_shard_4.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.10.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.10.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.10.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.10.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.10.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.10.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.11.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.11.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.11.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "d239bc682abdbce941474b60214d58e0" + }, + { + "dataPath": "params_shard_5.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.11.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.11.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.11.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.11.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.11.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.11.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.12.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.12.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.12.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "a868eee694d767a696d466bef2f721f9" + }, + { + "dataPath": "params_shard_6.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.12.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.12.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.12.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.12.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.12.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.12.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.13.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.13.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.13.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "de907f438ae37441e91dab7ae38b6dda" + }, + { + "dataPath": "params_shard_7.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.13.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.13.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.13.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.13.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.13.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.13.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.14.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.14.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.14.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "7031d3bedab43f799bd88bba097adb6a" + }, + { + "dataPath": "params_shard_8.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.14.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.14.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.14.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.14.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.14.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.14.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.15.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.15.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.15.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "676c762e8d2ddc78d807e1c580c8bbe8" + }, + { + "dataPath": "params_shard_9.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.15.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.15.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.15.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.15.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.15.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.15.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.16.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.16.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.16.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "686cd9a652286672b3a26f82b8bd5c32" + }, + { + "dataPath": "params_shard_10.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.16.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.16.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.16.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.16.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.16.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.16.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.17.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.17.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.17.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "bb41cf85ba43f0e456cd0c08052a7252" + }, + { + "dataPath": "params_shard_11.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.17.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.17.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.17.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.17.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.17.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.17.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.18.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.18.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.18.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "eab128ed411802760457cd396fedf2c1" + }, + { + "dataPath": "params_shard_12.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.18.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.18.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.18.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.18.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.18.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.18.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.19.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.19.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.19.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "4dfd40022fe9995a1f21a91252236e5c" + }, + { + "dataPath": "params_shard_13.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.19.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.19.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.19.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.19.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.19.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.19.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.2.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.2.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.2.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "bf1d0d2aca7958c87733ccb9ffd5b311" + }, + { + "dataPath": "params_shard_14.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.2.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.2.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.2.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.2.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.2.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.2.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.20.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.20.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.20.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "f16ae23454387d7cfdb704bbe2e775d9" + }, + { + "dataPath": "params_shard_15.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.20.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.20.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.20.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.20.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.20.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.20.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.21.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.21.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.21.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "60fee6ca8339e84fd7488cf7bbc8e267" + }, + { + "dataPath": "params_shard_16.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.21.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.21.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.21.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.21.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.21.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.21.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.22.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.22.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.22.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "dc5d011b29ce61a94386f470caa58cbd" + }, + { + "dataPath": "params_shard_17.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.22.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.22.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.22.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.22.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.22.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.22.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.23.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.23.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.23.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "76f3515b5b0cd038c2b5e82899e86cad" + }, + { + "dataPath": "params_shard_18.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.23.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.23.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.23.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.23.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.23.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.23.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.24.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.24.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.24.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "6fdd90c07ec9c6ffb718e613dedf581a" + }, + { + "dataPath": "params_shard_19.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.24.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.24.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.24.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.24.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.24.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.24.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.24.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.25.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.25.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.25.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "64f1286728f20f93c204a39807a7d1fd" + }, + { + "dataPath": "params_shard_20.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.25.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.25.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.25.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.25.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.25.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.25.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.26.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.26.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.26.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "a765b709f2082733cbb14d2f0f8eeaae" + }, + { + "dataPath": "params_shard_21.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.26.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.26.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.26.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.26.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.26.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.26.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.27.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.27.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.27.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "35d1b336a95738564a688a71177173ad" + }, + { + "dataPath": "params_shard_22.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.27.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.27.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.27.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.27.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.27.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.27.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.3.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.3.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.3.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "66b10100db0fc3488d9d6e10cb8923f4" + }, + { + "dataPath": "params_shard_23.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.3.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.3.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.3.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.3.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.3.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.3.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.4.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.4.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.4.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "76384ccd67f416e3f6877e7071c2e517" + }, + { + "dataPath": "params_shard_24.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.4.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.4.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.4.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.4.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.4.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.4.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.5.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.5.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.5.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "226253ce5818363029039f5dd85815df" + }, + { + "dataPath": "params_shard_25.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.5.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.5.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.5.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.5.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.5.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.5.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.6.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.6.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.6.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "713647246af38a89986e310c2ffa6ae8" + }, + { + "dataPath": "params_shard_26.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.6.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.6.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.6.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.6.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.6.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.6.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.7.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.7.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.7.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "a9eb68d151dd3b71334c7a3d05ff3501" + }, + { + "dataPath": "params_shard_27.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.7.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.7.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.7.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.7.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.7.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.7.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.8.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.8.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.8.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "6c0570028edbfa299e4730e1d402d1e4" + }, + { + "dataPath": "params_shard_28.bin", + "format": "raw-shard", + "nbytes": 26331136, + "records": [ + { + "name": "model.layers.8.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.8.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.8.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.8.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.8.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.8.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.layers.9.input_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + }, + { + "name": "model.layers.9.mlp.down_proj.q_weight", + "shape": [ + 1536, + 1120 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 6881280, + "byteOffset": 18589696 + }, + { + "name": "model.layers.9.mlp.down_proj.q_scale", + "shape": [ + 1536, + 280 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 860160, + "byteOffset": 25470976 + } + ], + "md5sum": "c5476addbcdb406fdd7a1899c77e5cf7" + }, + { + "dataPath": "params_shard_29.bin", + "format": "raw-shard", + "nbytes": 18589696, + "records": [ + { + "name": "model.layers.9.mlp.gate_up_proj.q_weight", + "shape": [ + 17920, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 13762560, + "byteOffset": 0 + }, + { + "name": "model.layers.9.mlp.gate_up_proj.q_scale", + "shape": [ + 17920, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 1720320, + "byteOffset": 13762560 + }, + { + "name": "model.layers.9.post_attention_layernorm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 15482880 + }, + { + "name": "model.layers.9.self_attn.c_attn.bias", + "shape": [ + 2048 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 4096, + "byteOffset": 15485952 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_weight", + "shape": [ + 2048, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1572864, + "byteOffset": 15490048 + }, + { + "name": "model.layers.9.self_attn.c_attn.q_scale", + "shape": [ + 2048, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 196608, + "byteOffset": 17062912 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_weight", + "shape": [ + 1536, + 192 + ], + "dtype": "uint32", + "format": "f32-to-bf16", + "nbytes": 1179648, + "byteOffset": 17259520 + }, + { + "name": "model.layers.9.self_attn.o_proj.q_scale", + "shape": [ + 1536, + 48 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 147456, + "byteOffset": 18439168 + }, + { + "name": "model.norm.weight", + "shape": [ + 1536 + ], + "dtype": "float16", + "format": "f32-to-bf16", + "nbytes": 3072, + "byteOffset": 18586624 + } + ], + "md5sum": "3ea6e81a6fe950e1a422c6102cb61115" + } + ] +} \ No newline at end of file