train model

scripts/model.yaml

@@ -108,26 +108,23 @@ eval:
   final_validation: false
 
 # Optimizer-related arguments
-optimizer: bitsandbytes.optim.PagedAdamW8bit
-
-# optimizer:
-#   class_path: torch.optim.AdamW
-#   # class_path: grokadamw.GrokAdamW
-#   # class_path: bitsandbytes.optim.PagedAdamW
-#   # class_path: bitsandbytes.optim.AdamW8bit
-#   # class_path: bitsandbytes.optim.PagedAdamW8bit
+optimizer:
+  # class_path: torch.optim.AdamW
+  # class_path: grokadamw.GrokAdamW
+  class_path: bitsandbytes.optim.AdamW8bit
+  # class_path: bitsandbytes.optim.PagedAdamW8bit
   
-#   init_args:
-#     #   (type: float, default: 0.001)
-#     lr: 5e-5
+  init_args:
+    #   (type: float, default: 0.001)
+    lr: 5e-5
     
-#     #   (type: float, default: 0.01)
-#     weight_decay: 0.1
+    #   (type: float, default: 0.01)
+    weight_decay: 0.1
     
-#     #   (type: tuple, default: (0.9,0.999))
-#     betas:
-#       - 0.9
-#       - 0.95
+    #   (type: tuple, default: (0.9,0.999))
+    betas:
+      - 0.9
+      - 0.95
 
 # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
 devices: auto