Upload axolotl_config_l6xu8_mg.yml

axolotl_config_l6xu8_mg.yml

@@ -0,0 +1,76 @@
+base_model: ./mistralai_Mistral-Small-24B-Base-2501/
+# optionally might have model_type or tokenizer_type
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+# Automatically upload checkpoint and final model to HF
+# hub_model_id: username/custom_model_name
+
+trust_remote_code: true
+
+wandb_project: SmallMistral-Reflex2
+wandb_entity:
+wandb_watch:
+wandb_name:
+wandb_log_model:
+
+load_in_8bit: false
+load_in_4bit: false
+strict: false
+
+model_config:
+  output_router_logits: true
+
+chat_template: tokenizer_default
+
+datasets:
+  - path: Undi95/R1-RP-ShareGPT3
+    type: chat_template
+    chat_template: tokenizer_default
+    field_messages: conversations
+    message_field_role: from
+    message_field_content: value
+    roles:
+      user: ["human", "user"]
+      assistant: ["gpt", "assistant"]
+      system: ["system"]
+      tool: ["tool"]
+
+dataset_prepared_path: last_run_prepared
+shuffle_merged_datasets: true
+val_set_size: 0.05
+output_dir: ./out
+
+sequence_len: 4096
+sample_packing: true
+pad_to_sequence_len: true
+
+gradient_accumulation_steps: 1
+micro_batch_size: 1
+num_epochs: 2
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 0.0000025
+
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+
+gradient_checkpointing: unsloth
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+save_total_limit: 6
+saves_per_epoch: 2
+save_steps:
+debug:
+deepspeed:
+weight_decay: 0.0
+fsdp:
+fsdp_config:
+save_safetensors: true