{
  "_name_or_path": "/mnt/dolphinfs/ssd_pool/docker/user/hadoop-vacv/lidengjie/projects/fsvd-llm/cache/llama-7b-hf/models/stage1-bf16-usv",
  "architectures": [
    "LlamaForCausalLM"
  ],
  "attention_bias": false,
  "attention_dropout": 0.0,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "hidden_act": "silu",
  "hidden_size": 4096,
  "initializer_range": 0.02,
  "intermediate_size": 11008,
  "linear_info": {
    "model.layers.0.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2318
    },
    "model.layers.0.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1267
    },
    "model.layers.0.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1251
    },
    "model.layers.0.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 212
    },
    "model.layers.0.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 670
    },
    "model.layers.0.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 178
    },
    "model.layers.0.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1109
    },
    "model.layers.1.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2370
    },
    "model.layers.1.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1712
    },
    "model.layers.1.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1748
    },
    "model.layers.1.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 375
    },
    "model.layers.1.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1250
    },
    "model.layers.1.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 376
    },
    "model.layers.1.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1195
    },
    "model.layers.10.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 1980
    },
    "model.layers.10.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1607
    },
    "model.layers.10.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2033
    },
    "model.layers.10.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 631
    },
    "model.layers.10.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1141
    },
    "model.layers.10.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 686
    },
    "model.layers.10.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1153
    },
    "model.layers.11.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2249
    },
    "model.layers.11.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1764
    },
    "model.layers.11.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2106
    },
    "model.layers.11.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 551
    },
    "model.layers.11.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1329
    },
    "model.layers.11.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 568
    },
    "model.layers.11.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1569
    },
    "model.layers.12.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2298
    },
    "model.layers.12.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1699
    },
    "model.layers.12.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2100
    },
    "model.layers.12.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 604
    },
    "model.layers.12.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1252
    },
    "model.layers.12.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 696
    },
    "model.layers.12.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1318
    },
    "model.layers.13.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2335
    },
    "model.layers.13.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1761
    },
    "model.layers.13.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2126
    },
    "model.layers.13.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 534
    },
    "model.layers.13.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 867
    },
    "model.layers.13.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 611
    },
    "model.layers.13.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1152
    },
    "model.layers.14.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2344
    },
    "model.layers.14.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1835
    },
    "model.layers.14.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2151
    },
    "model.layers.14.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 658
    },
    "model.layers.14.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1202
    },
    "model.layers.14.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 702
    },
    "model.layers.14.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1406
    },
    "model.layers.15.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2409
    },
    "model.layers.15.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1903
    },
    "model.layers.15.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2155
    },
    "model.layers.15.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 572
    },
    "model.layers.15.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1327
    },
    "model.layers.15.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 688
    },
    "model.layers.15.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1472
    },
    "model.layers.16.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 0
    },
    "model.layers.16.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1985
    },
    "model.layers.16.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2214
    },
    "model.layers.16.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 613
    },
    "model.layers.16.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1246
    },
    "model.layers.16.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 665
    },
    "model.layers.16.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1421
    },
    "model.layers.17.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 0
    },
    "model.layers.17.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1962
    },
    "model.layers.17.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2190
    },
    "model.layers.17.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 612
    },
    "model.layers.17.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1292
    },
    "model.layers.17.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 684
    },
    "model.layers.17.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1394
    },
    "model.layers.18.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 0
    },
    "model.layers.18.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1990
    },
    "model.layers.18.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2188
    },
    "model.layers.18.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 520
    },
    "model.layers.18.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1370
    },
    "model.layers.18.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 555
    },
    "model.layers.18.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1391
    },
    "model.layers.19.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 0
    },
    "model.layers.19.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1969
    },
    "model.layers.19.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2140
    },
    "model.layers.19.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 480
    },
    "model.layers.19.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 956
    },
    "model.layers.19.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 517
    },
    "model.layers.19.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1170
    },
    "model.layers.2.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 0
    },
    "model.layers.2.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1771
    },
    "model.layers.2.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1985
    },
    "model.layers.2.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 571
    },
    "model.layers.2.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1601
    },
    "model.layers.2.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 512
    },
    "model.layers.2.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1451
    },
    "model.layers.20.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 0
    },
    "model.layers.20.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1915
    },
    "model.layers.20.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2126
    },
    "model.layers.20.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 513
    },
    "model.layers.20.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1027
    },
    "model.layers.20.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 562
    },
    "model.layers.20.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1215
    },
    "model.layers.21.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2460
    },
    "model.layers.21.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1922
    },
    "model.layers.21.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2074
    },
    "model.layers.21.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 368
    },
    "model.layers.21.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 900
    },
    "model.layers.21.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 374
    },
    "model.layers.21.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1093
    },
    "model.layers.22.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2339
    },
    "model.layers.22.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1911
    },
    "model.layers.22.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2050
    },
    "model.layers.22.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 335
    },
    "model.layers.22.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 533
    },
    "model.layers.22.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 352
    },
    "model.layers.22.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 766
    },
    "model.layers.23.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2237
    },
    "model.layers.23.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1903
    },
    "model.layers.23.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2040
    },
    "model.layers.23.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 372
    },
    "model.layers.23.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 647
    },
    "model.layers.23.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 343
    },
    "model.layers.23.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 827
    },
    "model.layers.24.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2129
    },
    "model.layers.24.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1912
    },
    "model.layers.24.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2049
    },
    "model.layers.24.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 253
    },
    "model.layers.24.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 447
    },
    "model.layers.24.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 250
    },
    "model.layers.24.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 753
    },
    "model.layers.25.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2121
    },
    "model.layers.25.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1976
    },
    "model.layers.25.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2068
    },
    "model.layers.25.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 206
    },
    "model.layers.25.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 375
    },
    "model.layers.25.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 247
    },
    "model.layers.25.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 700
    },
    "model.layers.26.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2077
    },
    "model.layers.26.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1994
    },
    "model.layers.26.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2075
    },
    "model.layers.26.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 194
    },
    "model.layers.26.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 387
    },
    "model.layers.26.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 227
    },
    "model.layers.26.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 710
    },
    "model.layers.27.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2056
    },
    "model.layers.27.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2029
    },
    "model.layers.27.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2098
    },
    "model.layers.27.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 172
    },
    "model.layers.27.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 279
    },
    "model.layers.27.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 190
    },
    "model.layers.27.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 579
    },
    "model.layers.28.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2102
    },
    "model.layers.28.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2085
    },
    "model.layers.28.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2179
    },
    "model.layers.28.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 281
    },
    "model.layers.28.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 529
    },
    "model.layers.28.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 291
    },
    "model.layers.28.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 773
    },
    "model.layers.29.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2235
    },
    "model.layers.29.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2205
    },
    "model.layers.29.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2324
    },
    "model.layers.29.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 157
    },
    "model.layers.29.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 381
    },
    "model.layers.29.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 218
    },
    "model.layers.29.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 643
    },
    "model.layers.3.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2278
    },
    "model.layers.3.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2094
    },
    "model.layers.3.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2251
    },
    "model.layers.3.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 542
    },
    "model.layers.3.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 920
    },
    "model.layers.3.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 534
    },
    "model.layers.3.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1283
    },
    "model.layers.30.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2449
    },
    "model.layers.30.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2307
    },
    "model.layers.30.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2394
    },
    "model.layers.30.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 143
    },
    "model.layers.30.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 438
    },
    "model.layers.30.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 166
    },
    "model.layers.30.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 652
    },
    "model.layers.31.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 0
    },
    "model.layers.31.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2427
    },
    "model.layers.31.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 0
    },
    "model.layers.31.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 211
    },
    "model.layers.31.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 516
    },
    "model.layers.31.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 205
    },
    "model.layers.31.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 713
    },
    "model.layers.4.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2198
    },
    "model.layers.4.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1964
    },
    "model.layers.4.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2219
    },
    "model.layers.4.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 658
    },
    "model.layers.4.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1302
    },
    "model.layers.4.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 640
    },
    "model.layers.4.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1316
    },
    "model.layers.5.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2294
    },
    "model.layers.5.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1899
    },
    "model.layers.5.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2332
    },
    "model.layers.5.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 638
    },
    "model.layers.5.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1002
    },
    "model.layers.5.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 654
    },
    "model.layers.5.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1228
    },
    "model.layers.6.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2214
    },
    "model.layers.6.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1834
    },
    "model.layers.6.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2277
    },
    "model.layers.6.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 642
    },
    "model.layers.6.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1225
    },
    "model.layers.6.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 717
    },
    "model.layers.6.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1232
    },
    "model.layers.7.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2072
    },
    "model.layers.7.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1678
    },
    "model.layers.7.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2130
    },
    "model.layers.7.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 564
    },
    "model.layers.7.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1167
    },
    "model.layers.7.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 557
    },
    "model.layers.7.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1256
    },
    "model.layers.8.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2066
    },
    "model.layers.8.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1599
    },
    "model.layers.8.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2113
    },
    "model.layers.8.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 659
    },
    "model.layers.8.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1130
    },
    "model.layers.8.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 650
    },
    "model.layers.8.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1292
    },
    "model.layers.9.mlp.down_proj": {
      "bias": true,
      "in_features": 11008,
      "out_features": 4096,
      "rank": 2057
    },
    "model.layers.9.mlp.gate_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 1606
    },
    "model.layers.9.mlp.up_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 11008,
      "rank": 2054
    },
    "model.layers.9.self_attn.k_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 633
    },
    "model.layers.9.self_attn.o_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1003
    },
    "model.layers.9.self_attn.q_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 662
    },
    "model.layers.9.self_attn.v_proj": {
      "bias": true,
      "in_features": 4096,
      "out_features": 4096,
      "rank": 1049
    }
  },
  "max_position_embeddings": 2048,
  "mlp_bias": false,
  "model_type": "llama",
  "num_attention_heads": 32,
  "num_hidden_layers": 32,
  "num_key_value_heads": 32,
  "pad_token_id": 0,
  "pretraining_tp": 1,
  "rms_norm_eps": 1e-06,
  "rope_scaling": null,
  "rope_theta": 10000.0,
  "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.43.2",
  "use_cache": true,
  "vocab_size": 32000
}