TRI-ML
/

DCLM-1B-v0

Update config.json

by sedrickkeh - opened Jul 18

←

Files changed (1) hide show

config.json CHANGED Viewed

@@ -12,7 +12,7 @@
     "dim": 2048,
     "ffn_type": "swiglu_torch",
     "model": "open_lm_1b_swiglutorch",
-    "model_norm": "gain_only_lp_layer_norm",
     "moe_capacity_factor": 1.25,
     "moe_expert_model_parallelism": false,
     "moe_freq": 0,

     "dim": 2048,
     "ffn_type": "swiglu_torch",
     "model": "open_lm_1b_swiglutorch",
+    "norm_type": "gain_only_lp_layer_norm",
     "moe_capacity_factor": 1.25,
     "moe_expert_model_parallelism": false,
     "moe_freq": 0,