Training in progress, step 154

README.md

@@ -1,8 +1,7 @@
 ---
-base_model: google/gemma-2-27b-it
-license: gemma
 library_name: peft
-pipeline_tag: text-generation
+license: gemma
+base_model: google/gemma-2-27b-it
 tags:
 - axolotl
 - generated_from_trainer
@@ -21,7 +20,121 @@ should probably proofread and complete it, then remove this comment. -->
 
 axolotl version: `0.6.0`
 ```yaml
+# base_model: meta-llama/Llama-3.2-1B-Instruct
+# # Automatically upload checkpoint and final model to HF
+# # hub_model_id: kweinmeister/Llama-3.2-1B-Instruct-MetaMathQA
+# hub_model_id: kweinmeister/Llama-3.2-1B-Instruct-gsm8k
+
+# load_in_8bit: false
+# load_in_4bit: true
+# strict: false
+
+
+# datasets:
+#   - path: openai/gsm8k
+#     type: alpaca_chat.load_qa
+#     name: "main"
+#     train_on_split: "train"
+
+
+# # datasets:
+# #   - path: meta-math/MetaMathQA
+# #     type:
+# #       field_instruction: query
+# #       field_output: response
+
+# val_set_size: 0.1
+# # output_dir: "/mnt/disks/gcs/axolotl/outputs/out"
+# output_dir: "/mnt/disks/gcs/axolotl/outputs/gsm8k-out"
+# # output_dir: "/mnt/disks/gcs/axolotl/outputs/MetaMathQA-out"
+
+# adapter: qlora
+# lora_model_dir:
+
+# sequence_len: 2048
+# sample_packing: true
+# eval_sample_packing: true
+# pad_to_sequence_len: true
+
+# lora_r: 32
+# lora_alpha: 16
+# lora_dropout: 0.05
+# lora_fan_in_fan_out:
+# lora_target_modules:
+#   - gate_proj
+#   - down_proj
+#   - up_proj
+#   - q_proj
+#   - v_proj
+#   - k_proj
+#   - o_proj
+
+# wandb_project:
+# wandb_entity:
+# wandb_watch:
+# wandb_name:
+# wandb_log_model:
+
+# gradient_accumulation_steps: 4
+# micro_batch_size: 2
+# num_epochs: 3
+# # optimizer: adamw_bnb_8bit
+# optimizer: adamw_torch
+# lr_scheduler: cosine
+# learning_rate: 2e-5
+
+# train_on_inputs: false
+# group_by_length: false
+# bf16: auto
+# fp16:
+# tf32: false
+
+# # gradient_checkpointing: true
+# gradient_checkpointing: false
+# early_stopping_patience:
+# resume_from_checkpoint:
+# local_rank:
+# logging_steps: 1
+# xformers_attention:
+# flash_attention: true
+
+# loss_watchdog_threshold: 5.0
+# loss_watchdog_patience: 3
+
+# warmup_steps: 10
+# evals_per_epoch: 4
+# eval_table_size:
+# eval_max_new_tokens: 128
+# saves_per_epoch: 1
+# debug:
+# deepspeed:
+# weight_decay: 0.0
+# # fsdp:
+# # fsdp_config:
+# fsdp:
+#   - full_shard
+#   - auto_wrap  
+# fsdp_config:
+#   fsdp_limit_all_gathers: true
+#   fsdp_sync_module_states: true
+#   fsdp_offload_params: true
+#   fsdp_use_orig_params: false
+#   fsdp_cpu_ram_efficient_loading: true
+#   fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+#   fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
+#   fsdp_state_dict_type: FULL_STATE_DICT
+#   fsdp_sharding_strategy: FULL_SHARD
+#   fsdp_activation_checkpointing: true  
+# special_tokens:
+#   # pad_token: "<|end_of_text|>"
+#   special_tokens:
+#   bos_token: "<|begin_of_text|>"
+#   eos_token: "<|eot_id|>"
+#   pad_token: "<|finetune_right_pad_id|>"
+
 base_model: google/gemma-2-27b-it
+# model_type: AutoModelForCausalLM
+# tokenizer_type: AutoTokenizer
 hub_model_id: kweinmeister/gemma-2-27b-it-dolly-15k
 
 load_in_8bit: false
@@ -39,6 +152,7 @@ val_set_size: 0.1
 output_dir: "/mnt/disks/gcs/axolotl/outputs/dolly-15k-out"
 
 adapter: qlora
+
 lora_r: 32
 lora_alpha: 16
 lora_dropout: 0.05
@@ -46,21 +160,26 @@ lora_target_linear: true
 
 sequence_len: 2048
 sample_packing: true
+# eval_sample_packing: true
 pad_to_sequence_len: true
 
 gradient_accumulation_steps: 4
 micro_batch_size: 2
 num_epochs: 3
+# optimizer: adamw_bnb_8bit
 optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 2e-5
 
+
 train_on_inputs: false
 group_by_length: false
 bf16: auto
 fp16:
 tf32: false
 
+
+# gradient_checkpointing: false
 gradient_checkpointing: true
 early_stopping_patience:
 resume_from_checkpoint:
@@ -69,16 +188,56 @@ logging_steps: 1
 xformers_attention:
 flash_attention: false
 
+# loss_watchdog_threshold: 5.0
+# loss_watchdog_patience: 3
+
+
 warmup_ratio: 0.1
 evals_per_epoch: 4
 eval_max_new_tokens: 128
 saves_per_epoch: 1
 debug:
-deepspeed: deepspeed_configs/zero1.json
+# deepspeed:
 weight_decay: 0.0
 
+deepspeed: deepspeed_configs/zero1.json
+
 fsdp:
 fsdp_config:
+# fsdp:
+#   - full_shard
+#   - auto_wrap  
+
+# fsdp_config:
+#   fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+#   fsdp_backward_prefetch: BACKWARD_PRE
+#   fsdp_cpu_ram_efficient_loading: true
+#   fsdp_forward_prefetch: false
+#   fsdp_offload_params: true
+#   fsdp_sharding_strategy: FULL_SHARD
+#   fsdp_state_dict_type: SHARDED_STATE_DICT
+#   fsdp_transformer_layer_cls_to_wrap: GemmaDecoderLayer
+#   fsdp_sync_module_states: true
+#   fsdp_use_orig_params: true
+
+
+# fsdp_config:
+#   fsdp_limit_all_gathers: true
+#   fsdp_sync_module_states: true
+#   fsdp_offload_params: true
+#   fsdp_use_orig_params: false
+#   fsdp_cpu_ram_efficient_loading: true
+#   fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+#   fsdp_transformer_layer_cls_to_wrap: GemmaDecoderLayer
+#   fsdp_state_dict_type: FULL_STATE_DICT
+#   fsdp_sharding_strategy: FULL_SHARD
+#   fsdp_activation_checkpointing: true  
+# special_tokens:
+#   # pad_token: "<|end_of_text|>"
+#   special_tokens:
+#   bos_token: "<|begin_of_text|>"
+#   eos_token: "<|eot_id|>"
+#   pad_token: "<|finetune_right_pad_id|>"
 ```
 
 </details><br>

adapter_config.json

@@ -23,13 +23,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
     "gate_proj",
     "v_proj",
-    "k_proj",
-    "o_proj",
-    "q_proj",
     "down_proj",
-    "up_proj"
+    "k_proj",
+    "up_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5709a09cb3224a0ce11a226d4f596d5351cb9762a1b2a0eb00e00519c8b9f431
+oid sha256:58b08d330dcb589f67d630845c8d7b4e9398a45427d9eac63f91ba0b7f03406c
 size 456807968

config.json

@@ -27,7 +27,7 @@
     "_load_in_4bit": true,
     "_load_in_8bit": false,
     "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
     "bnb_4bit_use_double_quant": true,
     "llm_int8_enable_fp32_cpu_offload": false,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d1596f14357a7da8bd1d96e2ae56deec6db9b25132673be4df1cd5d1a18bc8e
+oid sha256:0a6b229a80811553a8917ed8e0f624c6e8ad3674fe3237b402de0aa44dd4c70d
 size 7992