Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+log.out filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250101_112144-t9wzg2aq/run-t9wzg2aq.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250102_021927-pw8rud5e/run-pw8rud5e.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250102_074844-1ecgrehs/run-1ecgrehs.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250103_132649-33elw7rf/run-33elw7rf.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250104_084255-r3b53lzn/run-r3b53lzn.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250117_195232-mv1zgbot/run-mv1zgbot.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250121_083313-ma4iz8vp/run-ma4iz8vp.wandb filter=lfs diff=lfs merge=lfs -text

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.0778410631950095,
+    "num_input_tokens_seen": 7732199424,
+    "train_loss": 0.036126901625228955,
+    "train_runtime": 9429.4679,
+    "train_samples_per_second": 0.782,
+    "train_steps_per_second": 0.391
+}

config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 128256
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.2"
+}

log.out

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6963529e7017bead51a7c8c4ec43da1c11e51fc7e7bf359f1c8bf8031e2887b
+size 11215004

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b37b547ddb30f3ba8c469637a2b3b2856ede8d588143d3d4cfa00f049136250
+size 4886466168

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63d61bd8d47275a3c52e65bfbcdce67cbec860de33955ef431ad924a91d5f57f
+size 4832007448

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f539fcd92b48eb4d6aa603aada5453477cb33d75b497bf64b38ae096dac750e9
+size 4999813112

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8649934669fba2177b102627fde9f1eaceb4c38931d33c37b3f063130823e5c2
+size 4999813128

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cc2c14df798d8bfdc66be730382828c9d04953f542dedd59dd6aa95fcea672d
+size 4832007496

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3f0a55d8f39643270fa55bef347acf3df13c5b47cec0cf10ab2beaa5cf30216
+size 4999813120

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1531d157bbcfe2a426999ef6b0d7b9ee73a11372433f6f22d067e1a801caa6ea
+size 2571158184

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32121044992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,2062 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.0778410631950095,
+    "num_input_tokens_seen": 7732199424,
+    "train_loss": 0.036126901625228955,
+    "train_runtime": 9429.4679,
+    "train_samples_per_second": 0.782,
+    "train_steps_per_second": 0.391
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0c16558cb168f94a8e3ea814f2ed1f2dd4c0ffd2157ce5165c1d9f7a30dce59
+size 5560

wandb/debug.log

@@ -0,0 +1,32 @@
+2025-01-21 08:33:13,169 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Current SDK version is 0.17.3
+2025-01-21 08:33:13,169 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Configure stats pid to 285553
+2025-01-21 08:33:13,169 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Loading settings from /home/ctpham_umass_edu/.config/wandb/settings
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Loading settings from /work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/wandb/settings
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'project': 'prolong', 'api_key': '***REDACTED***', 'mode': 'online'}
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'prolong-final/finetune.py', 'program_abspath': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py'}
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_init.py:_log_setup():520] Logging user logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250121_083313-ma4iz8vp/logs/debug.log
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_init.py:_log_setup():521] Logging internal logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250121_083313-ma4iz8vp/logs/debug-internal.log
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_init.py:init():560] calling init triggers
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_init.py:init():567] wandb.init called with sweep_config: {}
+config: {}
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_init.py:init():610] starting backend
+2025-01-21 08:33:13,170 INFO    MainThread:285553 [wandb_init.py:init():614] setting up manager
+2025-01-21 08:33:13,173 INFO    MainThread:285553 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-21 08:33:13,176 INFO    MainThread:285553 [wandb_init.py:init():622] backend started and connected
+2025-01-21 08:33:13,185 INFO    MainThread:285553 [wandb_init.py:init():711] updated telemetry
+2025-01-21 08:33:13,225 INFO    MainThread:285553 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout
+2025-01-21 08:33:13,495 INFO    MainThread:285553 [wandb_run.py:_on_init():2402] communicating current version
+2025-01-21 08:33:13,562 INFO    MainThread:285553 [wandb_run.py:_on_init():2411] got version response upgrade_message: "wandb version 0.19.4 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-21 08:33:13,562 INFO    MainThread:285553 [wandb_init.py:init():795] starting run threads in backend
+2025-01-21 08:33:20,849 INFO    MainThread:285553 [wandb_run.py:_console_start():2380] atexit reg
+2025-01-21 08:33:20,849 INFO    MainThread:285553 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2025-01-21 08:33:20,849 INFO    MainThread:285553 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2025-01-21 08:33:20,850 INFO    MainThread:285553 [wandb_run.py:_redirect():2325] Redirects installed.
+2025-01-21 08:33:20,851 INFO    MainThread:285553 [wandb_init.py:init():838] run started, returning control to user process
+2025-01-21 08:33:20,853 INFO    MainThread:285553 [wandb_run.py:_config_callback():1382] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 32, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': 0, 'eos_token_id': [128001, 128008, 128009], 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', 'transformers_version': '4.44.2', 'model_type': 'llama', 'output_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'overwrite_output_dir': False, 'do_train': True, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-06, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.05, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/runs/Jan21_08-26-17_gpu021', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1.0, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 200, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 1, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'disable_tqdm': True, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': True, 'fsdp': ['auto_wrap', 'offload'], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'eval_use_gather_object': False, 'min_lr_ratio': 0.1, 'cuda_empty_cache': True, 'streaming_dataset': True, 'seq_parallel_size': 8}
+2025-01-21 08:33:20,857 INFO    MainThread:285553 [wandb_config.py:__setitem__():151] config set model/num_parameters = 1003782656 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x72984c36eda0>>
+2025-01-21 08:33:20,857 INFO    MainThread:285553 [wandb_run.py:_config_callback():1382] config_cb model/num_parameters 1003782656 None
+2025-01-21 11:11:55,795 WARNING MsgRouterThr:285553 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241231_100054-t2idt8o9/files/conda-environment.yaml

@@ -0,0 +1,233 @@
+name: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - bzip2=1.0.8=h4bc722e_7
+  - ca-certificates=2024.12.14=hbcca054_0
+  - ld_impl_linux-64=2.43=h712a8e2_2
+  - libffi=3.4.2=h7f98852_5
+  - libgcc=14.2.0=h77fa898_1
+  - libgcc-ng=14.2.0=h69a702a_1
+  - libgomp=14.2.0=h77fa898_1
+  - liblzma=5.6.3=hb9d3cd8_1
+  - liblzma-devel=5.6.3=hb9d3cd8_1
+  - libnsl=2.0.1=hd590300_0
+  - libsqlite=3.47.2=hee588c1_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.3.1=hb9d3cd8_2
+  - ncurses=6.5=he02047a_1
+  - openssl=3.4.0=hb9d3cd8_0
+  - pip=24.3.1=pyh8b19718_2
+  - python=3.10.0=h543edf9_3_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=75.6.0=pyhff2d567_1
+  - sqlite=3.47.2=h9eae976_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - wheel=0.45.1=pyhd8ed1ab_1
+  - xz=5.6.3=hbcc6ac9_1
+  - xz-gpl-tools=5.6.3=hbcc6ac9_1
+  - xz-tools=5.6.3=hb9d3cd8_1
+  - pip:
+      - accelerate==0.32.1
+      - aiohappyeyeballs==2.4.3
+      - aiohttp==3.11.2
+      - aioprometheus==23.12.0
+      - aiosignal==1.3.1
+      - annotated-types==0.7.0
+      - anthropic==0.39.0
+      - anyio==4.6.2.post1
+      - argcomplete==3.5.1
+      - arrow==1.3.0
+      - async-timeout==5.0.1
+      - attrs==24.2.0
+      - azure-core==1.32.0
+      - azure-identity==1.19.0
+      - azure-storage-blob==12.24.0
+      - azure-storage-file-datalake==12.18.0
+      - backoff==2.2.1
+      - bcrypt==4.2.0
+      - blobfile==3.0.0
+      - boto3==1.35.63
+      - botocore==1.35.63
+      - brotli==1.1.0
+      - cachetools==5.5.0
+      - certifi==2024.8.30
+      - cffi==1.17.1
+      - charset-normalizer==3.4.0
+      - circuitbreaker==2.0.0
+      - click==8.1.7
+      - cloudpickle==3.1.0
+      - compressed-tensors==0.8.0
+      - contourpy==1.3.1
+      - cramjam==2.9.0
+      - cryptography==43.0.3
+      - cycler==0.12.1
+      - datasets==2.20.0
+      - debugpy==1.8.11
+      - dill==0.3.8
+      - diskcache==5.6.3
+      - distro==1.9.0
+      - docker-pycreds==0.4.0
+      - docstring-parser==0.16
+      - einops==0.8.0
+      - fastapi==0.115.5
+      - filelock==3.16.1
+      - flash-attn==2.6.1
+      - fonttools==4.55.0
+      - frozenlist==1.5.0
+      - fsspec==2024.5.0
+      - gguf==0.10.0
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - google-api-core==2.23.0
+      - google-auth==2.36.0
+      - google-cloud-aiplatform==1.71.1
+      - google-cloud-bigquery==3.27.0
+      - google-cloud-core==2.4.1
+      - google-cloud-resource-manager==1.13.1
+      - google-cloud-storage==2.10.0
+      - google-crc32c==1.6.0
+      - google-resumable-media==2.7.2
+      - googleapis-common-protos==1.66.0
+      - gql==3.5.0
+      - graphql-core==3.2.5
+      - grpc-google-iam-v1==0.13.1
+      - grpcio==1.68.0
+      - grpcio-status==1.62.3
+      - h11==0.14.0
+      - httpcore==1.0.7
+      - httptools==0.6.4
+      - httpx==0.27.2
+      - huggingface-hub==0.26.2
+      - idna==3.10
+      - importlib-metadata==8.5.0
+      - interegular==0.3.3
+      - ipython==8.18.0
+      - isodate==0.7.2
+      - jedi==0.19.2
+      - jinja2==3.1.4
+      - jiter==0.7.1
+      - jmespath==1.0.1
+      - jsonschema==4.23.0
+      - jsonschema-specifications==2024.10.1
+      - kiwisolver==1.4.7
+      - lark==1.2.2
+      - llvmlite==0.43.0
+      - lm-format-enforcer==0.10.9
+      - lxml==5.3.0
+      - markdown-it-py==3.0.0
+      - markupsafe==3.0.2
+      - matplotlib==3.9.2
+      - mdurl==0.1.2
+      - mosaicml-cli==0.5.34
+      - mosaicml-streaming==0.8.1
+      - mpmath==1.3.0
+      - msal==1.31.1
+      - msal-extensions==1.2.0
+      - msgpack==1.1.0
+      - msgspec==0.18.6
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - networkx==3.4.2
+      - ninja==1.11.1.1
+      - numba==0.60.0
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==9.1.0.70
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.4.127
+      - nvidia-nvtx-cu12==12.1.105
+      - oci==2.138.1
+      - openai==1.54.5
+      - opencv-python-headless==4.10.0.84
+      - orjson==3.10.11
+      - outlines==0.0.46
+      - packaging==24.1
+      - pandas==2.2.1
+      - paramiko==3.5.0
+      - partial-json-parser==0.2.1.1.post4
+      - pillow==10.4.0
+      - portalocker==2.10.1
+      - prometheus-client==0.21.0
+      - prometheus-fastapi-instrumentator==7.0.0
+      - prompt-toolkit==3.0.36
+      - propcache==0.2.0
+      - proto-plus==1.25.0
+      - protobuf==4.25.3
+      - py-cpuinfo==9.0.0
+      - pyairports==2.1.1
+      - pyarrow==18.0.0
+      - pyarrow-hotfix==0.6
+      - pyasn1==0.6.1
+      - pyasn1-modules==0.4.1
+      - pycountry==24.6.1
+      - pycparser==2.22
+      - pycryptodomex==3.21.0
+      - pydantic==2.9.2
+      - pydantic-core==2.23.4
+      - pyjwt==2.10.0
+      - pynacl==1.5.0
+      - pyopenssl==24.2.1
+      - pyparsing==3.2.0
+      - python-dateutil==2.9.0
+      - python-dotenv==1.0.1
+      - python-snappy==0.7.3
+      - pytz==2024.2
+      - pyyaml==6.0.2
+      - quantile-python==1.1
+      - questionary==2.0.1
+      - ray==2.39.0
+      - referencing==0.35.1
+      - regex==2023.12.25
+      - requests==2.32.3
+      - rich==13.9.4
+      - rotary-emb==0.5.2
+      - rpds-py==0.21.0
+      - rsa==4.9
+      - ruamel-yaml==0.18.6
+      - ruamel-yaml-clib==0.2.12
+      - s3transfer==0.10.3
+      - safetensors==0.4.5
+      - sentencepiece==0.1.99
+      - sentry-sdk==2.18.0
+      - setproctitle==1.3.4
+      - shapely==2.0.6
+      - simple-parsing==0.1.6
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - starlette==0.41.3
+      - sympy==1.13.1
+      - tiktoken==0.7.0
+      - tokenizers==0.19.1
+      - torch==2.4.1
+      - torchvision==0.19.1
+      - tqdm==4.66.4
+      - transformers==4.44.2
+      - triton==3.0.0
+      - types-python-dateutil==2.9.0.20241003
+      - tzdata==2024.2
+      - urllib3==2.2.3
+      - uvicorn==0.32.0
+      - uvloop==0.21.0
+      - validators==0.34.0
+      - vertexai==1.71.1
+      - wandb==0.17.3
+      - watchfiles==0.24.0
+      - websockets==11.0.3
+      - xformers==0.0.28.post1
+      - xxhash==3.5.0
+      - yarl==1.17.2
+      - zipp==3.21.0
+      - zstandard==0.23.0
+      - zstd==1.5.5.1
+prefix: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final

wandb/run-20241231_100054-t2idt8o9/files/config.yaml

@@ -0,0 +1,713 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.0
+    cli_version: 0.17.3
+    framework: huggingface
+    huggingface_version: 4.44.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735639254
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 105
+      3:
+      - 7
+      - 13
+      - 19
+      - 23
+      - 66
+      4: 3.10.0
+      5: 0.17.3
+      6: 4.44.2
+      8:
+      - 5
+      9:
+        1: transformers_trainer
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+    - 1: train/loss
+      5: 1
+      6:
+      - 1
+    - 1: train/grad_norm
+      5: 1
+      6:
+      - 1
+    - 1: train/learning_rate
+      5: 1
+      6:
+      - 1
+    - 1: train/epoch
+      5: 1
+      6:
+      - 1
+    - 1: train/num_input_tokens_seen
+      5: 1
+      6:
+      - 1
+vocab_size:
+  desc: null
+  value: 128256
+max_position_embeddings:
+  desc: null
+  value: 131072
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 14336
+num_hidden_layers:
+  desc: null
+  value: 32
+num_attention_heads:
+  desc: null
+  value: 32
+num_key_value_heads:
+  desc: null
+  value: 8
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-05
+pretraining_tp:
+  desc: null
+  value: 1
+use_cache:
+  desc: null
+  value: true
+rope_theta:
+  desc: null
+  value: 500000.0
+rope_scaling:
+  desc: null
+  value:
+    factor: 8.0
+    low_freq_factor: 1.0
+    high_freq_factor: 4.0
+    original_max_position_embeddings: 8192
+    rope_type: llama3
+attention_bias:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.0
+mlp_bias:
+  desc: null
+  value: false
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 128000
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value:
+  - 128001
+  - 128008
+  - 128009
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: /datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/
+transformers_version:
+  desc: null
+  value: 4.44.2
+model_type:
+  desc: null
+  value: llama
+output_dir:
+  desc: null
+  value: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: true
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+eval_strategy:
+  desc: null
+  value: 'no'
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 1
+per_device_eval_batch_size:
+  desc: null
+  value: 8
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+gradient_accumulation_steps:
+  desc: null
+  value: 2
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_delay:
+  desc: null
+  value: 0
+torch_empty_cache_steps:
+  desc: null
+  value: null
+learning_rate:
+  desc: null
+  value: 1.0e-06
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 1.0
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+lr_scheduler_kwargs:
+  desc: null
+  value: {}
+warmup_ratio:
+  desc: null
+  value: 0.05
+warmup_steps:
+  desc: null
+  value: 0
+log_level:
+  desc: null
+  value: info
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/runs/Dec31_10-00-19_gpu020
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 1.0
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: steps
+save_steps:
+  desc: null
+  value: 200
+save_total_limit:
+  desc: null
+  value: null
+save_safetensors:
+  desc: null
+  value: true
+save_on_each_node:
+  desc: null
+  value: false
+save_only_model:
+  desc: null
+  value: false
+restore_callback_states_from_checkpoint:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_cpu:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: null
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: null
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: null
+tpu_num_cores:
+  desc: null
+  value: null
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: []
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: null
+dataloader_num_workers:
+  desc: null
+  value: 1
+dataloader_prefetch_factor:
+  desc: null
+  value: null
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: _llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_
+disable_tqdm:
+  desc: null
+  value: true
+remove_unused_columns:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: null
+greater_is_better:
+  desc: null
+  value: null
+ignore_data_skip:
+  desc: null
+  value: false
+fsdp:
+  desc: null
+  value:
+  - auto_wrap
+  - offload
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value:
+    min_num_params: 0
+    xla: false
+    xla_fsdp_v2: false
+    xla_fsdp_grad_ckpt: false
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: null
+accelerator_config:
+  desc: null
+  value:
+    split_batches: false
+    dispatch_batches: null
+    even_batches: true
+    use_seedable_sampler: true
+    non_blocking: false
+    gradient_accumulation_kwargs: null
+deepspeed:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: null
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value:
+  - wandb
+ddp_find_unused_parameters:
+  desc: null
+  value: false
+ddp_bucket_cap_mb:
+  desc: null
+  value: null
+ddp_broadcast_buffers:
+  desc: null
+  value: null
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataloader_persistent_workers:
+  desc: null
+  value: false
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: false
+resume_from_checkpoint:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: null
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+hub_always_push:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+gradient_checkpointing_kwargs:
+  desc: null
+  value: null
+include_inputs_for_metrics:
+  desc: null
+  value: false
+eval_do_concat_batches:
+  desc: null
+  value: true
+fp16_backend:
+  desc: null
+  value: auto
+evaluation_strategy:
+  desc: null
+  value: null
+push_to_hub_model_id:
+  desc: null
+  value: null
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: null
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: null
+torch_compile_mode:
+  desc: null
+  value: null
+dispatch_batches:
+  desc: null
+  value: null
+split_batches:
+  desc: null
+  value: null
+include_tokens_per_second:
+  desc: null
+  value: false
+include_num_input_tokens_seen:
+  desc: null
+  value: false
+neftune_noise_alpha:
+  desc: null
+  value: null
+optim_target_modules:
+  desc: null
+  value: null
+batch_eval_metrics:
+  desc: null
+  value: false
+eval_on_start:
+  desc: null
+  value: false
+eval_use_gather_object:
+  desc: null
+  value: false
+min_lr_ratio:
+  desc: null
+  value: 0.1
+cuda_empty_cache:
+  desc: null
+  value: true
+streaming_dataset:
+  desc: null
+  value: true
+seq_parallel_size:
+  desc: null
+  value: 8
+model/num_parameters:
+  desc: null
+  value: 1003782656

wandb/run-20241231_100054-t2idt8o9/files/output.log

@@ -0,0 +1,4 @@
+/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context:  # type: ignore[attr-defined]
+[INFO|trainer.py:175] 2024-12-31 10:01:50,057 >> {'loss': 1.4933, 'grad_norm': 33.300479888916016, 'learning_rate': 5.405405405405406e-09, 'epoch': 0.00027122321670735016, 'num_input_tokens_seen': 2097152, 'completed': '0.03% (1 / 3_687)', 'remaining time': '49:07:09', 'throughput': '2732.19', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2024-12-31 10:02:25,461 >> {'loss': 1.6295, 'grad_norm': 35.11330795288086, 'learning_rate': 1.0810810810810811e-08, 'epoch': 0.0005424464334147003, 'num_input_tokens_seen': 4194304, 'completed': '0.05% (2 / 3_687)', 'remaining time': '42:40:22', 'throughput': '7404.33', 'gpu_mem_free': '5581MB'}

wandb/run-20241231_100054-t2idt8o9/files/requirements.txt

@@ -0,0 +1,244 @@
+Brotli==1.1.0
+GitPython==3.1.43
+Jinja2==3.1.4
+MarkupSafe==3.0.2
+PyJWT==2.10.0
+PyNaCl==1.5.0
+PyYAML==6.0.2
+Pygments==2.18.0
+accelerate==0.32.1
+aiohappyeyeballs==2.4.3
+aiohttp==3.11.2
+aioprometheus==23.12.0
+aiosignal==1.3.1
+annotated-types==0.7.0
+anthropic==0.39.0
+anyio==4.6.2.post1
+argcomplete==3.5.1
+arrow==1.3.0
+asttokens==2.4.1
+async-timeout==5.0.1
+attrs==24.2.0
+autocommand==2.2.2
+azure-core==1.32.0
+azure-identity==1.19.0
+azure-storage-blob==12.24.0
+azure-storage-file-datalake==12.18.0
+backoff==2.2.1
+backports.tarfile==1.2.0
+bcrypt==4.2.0
+blobfile==3.0.0
+boto3==1.35.63
+botocore==1.35.63
+cachetools==5.5.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.4.0
+circuitbreaker==2.0.0
+click==8.1.7
+cloudpickle==3.1.0
+comm==0.2.2
+compressed-tensors==0.8.0
+contourpy==1.3.1
+cramjam==2.9.0
+cryptography==43.0.3
+cycler==0.12.1
+datasets==2.20.0
+datatools==0.1
+debugpy==1.8.11
+decorator==5.1.1
+dill==0.3.8
+diskcache==5.6.3
+distro==1.9.0
+docker-pycreds==0.4.0
+docstring_parser==0.16
+einops==0.8.0
+exceptiongroup==1.2.2
+executing==2.1.0
+fastapi==0.115.5
+filelock==3.16.1
+flash-attn==2.6.1
+fonttools==4.55.0
+frozenlist==1.5.0
+fsspec==2024.5.0
+gguf==0.10.0
+gitdb==4.0.11
+google-api-core==2.23.0
+google-auth==2.36.0
+google-cloud-aiplatform==1.71.1
+google-cloud-bigquery==3.27.0
+google-cloud-core==2.4.1
+google-cloud-resource-manager==1.13.1
+google-cloud-storage==2.10.0
+google-crc32c==1.6.0
+google-resumable-media==2.7.2
+googleapis-common-protos==1.66.0
+gql==3.5.0
+graphql-core==3.2.5
+grpc-google-iam-v1==0.13.1
+grpcio-status==1.62.3
+grpcio==1.68.0
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.27.2
+huggingface-hub==0.26.2
+idna==3.10
+importlib_metadata==8.0.0
+importlib_metadata==8.5.0
+inflect==7.3.1
+interegular==0.3.3
+ipykernel==6.29.5
+ipython==8.18.0
+isodate==0.7.2
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jedi==0.19.2
+jiter==0.7.1
+jmespath==1.0.1
+jsonschema-specifications==2024.10.1
+jsonschema==4.23.0
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+kiwisolver==1.4.7
+lark==1.2.2
+llvmlite==0.43.0
+lm-format-enforcer==0.10.9
+lxml==5.3.0
+markdown-it-py==3.0.0
+matplotlib-inline==0.1.7
+matplotlib==3.9.2
+mdurl==0.1.2
+more-itertools==10.3.0
+mosaicml-cli==0.5.34
+mosaicml-streaming==0.8.1
+mpmath==1.3.0
+msal-extensions==1.2.0
+msal==1.31.1
+msgpack==1.1.0
+msgspec==0.18.6
+multidict==6.1.0
+multiprocess==0.70.16
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.11.1.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py==12.560.30
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+oci==2.138.1
+openai==1.54.5
+opencv-python-headless==4.10.0.84
+orjson==3.10.11
+outlines==0.0.46
+packaging==24.1
+packaging==24.2
+pandas==2.2.1
+paramiko==3.5.0
+parso==0.8.4
+partial-json-parser==0.2.1.1.post4
+pexpect==4.9.0
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+platformdirs==4.3.6
+portalocker==2.10.1
+prometheus-fastapi-instrumentator==7.0.0
+prometheus_client==0.21.0
+prompt-toolkit==3.0.36
+propcache==0.2.0
+proto-plus==1.25.0
+protobuf==4.25.3
+psutil==6.1.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py-cpuinfo==9.0.0
+pyOpenSSL==24.2.1
+pyairports==2.1.1
+pyarrow-hotfix==0.6
+pyarrow==18.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pycountry==24.6.1
+pycparser==2.22
+pycryptodomex==3.21.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+pyparsing==3.2.0
+python-dateutil==2.9.0
+python-dotenv==1.0.1
+python-snappy==0.7.3
+pytz==2024.2
+pyzmq==26.2.0
+quantile-python==1.1
+questionary==2.0.1
+ray==2.39.0
+referencing==0.35.1
+regex==2023.12.25
+requests==2.32.3
+rich==13.9.4
+rotary-emb==0.5.2
+rpds-py==0.21.0
+rsa==4.9
+ruamel.yaml.clib==0.2.12
+ruamel.yaml==0.18.6
+s3transfer==0.10.3
+safetensors==0.4.5
+sentencepiece==0.1.99
+sentry-sdk==2.18.0
+setproctitle==1.3.4
+setuptools==75.6.0
+shapely==2.0.6
+simple-parsing==0.1.6
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+stack-data==0.6.3
+starlette==0.41.3
+sympy==1.13.1
+tiktoken==0.7.0
+tokenizers==0.19.1
+tomli==2.0.1
+torch==2.4.1
+torchvision==0.19.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.44.2
+triton==3.0.0
+typeguard==4.3.0
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+typing_extensions==4.12.2
+tzdata==2024.2
+urllib3==2.2.3
+uvicorn==0.32.0
+uvloop==0.21.0
+validators==0.34.0
+vertexai==1.71.1
+wandb==0.17.3
+watchfiles==0.24.0
+wcwidth==0.2.13
+websockets==11.0.3
+wheel==0.43.0
+wheel==0.45.1
+xformers==0.0.28.post1
+xxhash==3.5.0
+yarl==1.17.2
+zipp==3.19.2
+zipp==3.21.0
+zstandard==0.23.0
+zstd==1.5.5.1

wandb/run-20241231_100054-t2idt8o9/files/wandb-metadata.json

@@ -0,0 +1,705 @@
+{
+    "os": "Linux-6.8.0-48-generic-x86_64-with-glibc2.39",
+    "python": "3.10.0",
+    "heartbeatAt": "2024-12-31T10:00:55.248811",
+    "startedAt": "2024-12-31T10:00:54.766527",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_family",
+        "llama",
+        "--apply_instruct_masks",
+        "--token_scaled_loss",
+        "--seq_parallel_size",
+        "8",
+        "--report_to",
+        "wandb",
+        "--do_train",
+        "--model_name_or_path",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--config_name",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--tokenizer_name",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--run_name",
+        "_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_",
+        "--output_dir",
+        "/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_",
+        "--config_overrides_json",
+        "",
+        "--gradient_accumulation_steps",
+        "2",
+        "--per_device_train_batch_size",
+        "1",
+        "--bf16",
+        "--learning_rate",
+        "1e-6",
+        "--min_lr_ratio",
+        "0.1",
+        "--lr_scheduler_type",
+        "cosine",
+        "--max_grad_norm",
+        "1.0",
+        "--adam_beta1",
+        "0.9",
+        "--adam_beta2",
+        "0.95",
+        "--weight_decay",
+        "0.1",
+        "--warmup_ratio",
+        "0.05",
+        "--optim",
+        "adamw_torch",
+        "--logging_steps",
+        "1",
+        "--log_level",
+        "info",
+        "--save_steps",
+        "200",
+        "--dataloader_num_workers",
+        "1",
+        "--disable_tqdm",
+        "true",
+        "--use_fast_tokenizer",
+        "false",
+        "--remove_unused_columns",
+        "false",
+        "--ddp_find_unused_parameters",
+        "false",
+        "--fsdp",
+        "auto_wrap offload",
+        "--gradient_checkpointing",
+        "--tokenized_mds_train",
+        "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete",
+        "--cuda_empty_cache",
+        "--num_train_epochs",
+        "1"
+    ],
+    "state": "running",
+    "program": "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py",
+    "codePathLocal": "finetune.py",
+    "codePath": "prolong-final/finetune.py",
+    "git": {
+        "remote": "https://github.com/chtmp223/BookGen-dev.git",
+        "commit": "0e796521430a0f767be7c4dadba5c2fcaee1f909"
+    },
+    "email": "[email protected]",
+    "root": "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev",
+    "host": "gpu020",
+    "username": "ctpham_umass_edu",
+    "executable": "/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10",
+    "cpu_count": 112,
+    "cpu_count_logical": 112,
+    "cpu_freq": {
+        "current": 957.8740624999999,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2463.059,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 759.577,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2024.63,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2400.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1740.158,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1100.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2035.036,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1400.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1900.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 790.179,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1393.77,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1439.809,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 782.956,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 783.716,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 783.035,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.273,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.246,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.059,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 790.704,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 438.487850189209,
+            "used": 18.201271057128906
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-80GB",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        }
+    ],
+    "memory": {
+        "total": 2015.3287239074707
+    }
+}

wandb/run-20241231_100054-t2idt8o9/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss": 1.738, "train/grad_norm": 41.506248474121094, "train/learning_rate": 1.6216216216216218e-08, "train/epoch": 0.0008136696501220504, "train/num_input_tokens_seen": 6291456, "train/global_step": 3, "_timestamp": 1735639380.6717923, "_runtime": 125.88794016838074, "_step": 2}

wandb/run-20241231_100054-t2idt8o9/logs/debug-internal.log

@@ -0,0 +1,244 @@
+2024-12-31 10:00:54,781 INFO    StreamThr :1511293 [internal.py:wandb_internal():85] W&B internal server running at pid: 1511293, started at: 2024-12-31 10:00:54.780041
+2024-12-31 10:00:54,783 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status
+2024-12-31 10:00:54,789 INFO    WriterThread:1511293 [datastore.py:open_for_write():87] open: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/run-t2idt8o9.wandb
+2024-12-31 10:00:54,790 DEBUG   SenderThread:1511293 [sender.py:send():379] send: header
+2024-12-31 10:00:54,825 DEBUG   SenderThread:1511293 [sender.py:send():379] send: run
+2024-12-31 10:00:55,057 INFO    SenderThread:1511293 [dir_watcher.py:__init__():211] watching files in: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files
+2024-12-31 10:00:55,057 INFO    SenderThread:1511293 [sender.py:_start_run_threads():1188] run started: t2idt8o9 with start time 1735639254.783852
+2024-12-31 10:00:55,080 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 10:00:55,081 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: check_version
+2024-12-31 10:00:55,155 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 10:00:55,183 DEBUG   HandlerThread:1511293 [system_info.py:__init__():26] System info init
+2024-12-31 10:00:55,183 DEBUG   HandlerThread:1511293 [system_info.py:__init__():41] System info init done
+2024-12-31 10:00:55,183 INFO    HandlerThread:1511293 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 10:00:55,183 INFO    SystemMonitor:1511293 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 10:00:55,183 INFO    HandlerThread:1511293 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 10:00:55,184 INFO    SystemMonitor:1511293 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 10:00:55,185 INFO    SystemMonitor:1511293 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 10:00:55,186 INFO    SystemMonitor:1511293 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 10:00:55,186 INFO    SystemMonitor:1511293 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 10:00:55,187 INFO    SystemMonitor:1511293 [interfaces.py:start():188] Started network monitoring
+2024-12-31 10:00:55,248 DEBUG   HandlerThread:1511293 [system_info.py:probe():152] Probing system
+2024-12-31 10:00:55,251 DEBUG   HandlerThread:1511293 [system_info.py:_probe_git():137] Probing git
+2024-12-31 10:00:55,270 DEBUG   HandlerThread:1511293 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 10:00:55,270 DEBUG   HandlerThread:1511293 [system_info.py:probe():200] Probing system done
+2024-12-31 10:00:55,271 DEBUG   HandlerThread:1511293 [system_monitor.py:probe():223] {'os': 'Linux-6.8.0-48-generic-x86_64-with-glibc2.39', 'python': '3.10.0', 'heartbeatAt': '2024-12-31T10:00:55.248811', 'startedAt': '2024-12-31T10:00:54.766527', 'docker': None, 'cuda': None, 'args': ('--model_family', 'llama', '--apply_instruct_masks', '--token_scaled_loss', '--seq_parallel_size', '8', '--report_to', 'wandb', '--do_train', '--model_name_or_path', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--config_name', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--tokenizer_name', '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', '--run_name', '_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', '--output_dir', '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', '--config_overrides_json', '', '--gradient_accumulation_steps', '2', '--per_device_train_batch_size', '1', '--bf16', '--learning_rate', '1e-6', '--min_lr_ratio', '0.1', '--lr_scheduler_type', 'cosine', '--max_grad_norm', '1.0', '--adam_beta1', '0.9', '--adam_beta2', '0.95', '--weight_decay', '0.1', '--warmup_ratio', '0.05', '--optim', 'adamw_torch', '--logging_steps', '1', '--log_level', 'info', '--save_steps', '200', '--dataloader_num_workers', '1', '--disable_tqdm', 'true', '--use_fast_tokenizer', 'false', '--remove_unused_columns', 'false', '--ddp_find_unused_parameters', 'false', '--fsdp', 'auto_wrap offload', '--gradient_checkpointing', '--tokenized_mds_train', '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete', '--cuda_empty_cache', '--num_train_epochs', '1'), 'state': 'running', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'codePathLocal': 'finetune.py', 'codePath': 'prolong-final/finetune.py', 'git': {'remote': 'https://github.com/chtmp223/BookGen-dev.git', 'commit': '0e796521430a0f767be7c4dadba5c2fcaee1f909'}, 'email': '[email protected]', 'root': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev', 'host': 'gpu020', 'username': 'ctpham_umass_edu', 'executable': '/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10', 'cpu_count': 112, 'cpu_count_logical': 112, 'cpu_freq': {'current': 957.8740624999999, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 2463.059, 'min': 800.0, 'max': 3800.0}, {'current': 759.577, 'min': 800.0, 'max': 3800.0}, {'current': 2024.63, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1740.158, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1100.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2035.036, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1400.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1900.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 790.179, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1393.77, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 1439.809, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 782.956, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.716, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 783.035, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.273, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.246, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 784.059, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 790.704, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 438.487850189209, 'used': 18.201271057128906}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.3287239074707}}
+2024-12-31 10:00:55,271 INFO    HandlerThread:1511293 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 10:00:55,271 INFO    HandlerThread:1511293 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 10:00:55,271 DEBUG   HandlerThread:1511293 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2024-12-31 10:00:56,059 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/conda-environment.yaml
+2024-12-31 10:01:01,448 DEBUG   HandlerThread:1511293 [system_info.py:_save_conda():224] Saving conda packages done
+2024-12-31 10:01:01,452 INFO    HandlerThread:1511293 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 10:01:01,459 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:01,459 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 10:01:01,460 DEBUG   SenderThread:1511293 [sender.py:send():379] send: files
+2024-12-31 10:01:01,460 INFO    SenderThread:1511293 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 10:01:01,752 INFO    wandb-upload_0:1511293 [upload_job.py:push():130] Uploaded file /tmp/tmp_luoc01bwandb/35p6d7uj-wandb-metadata.json
+2024-12-31 10:01:01,779 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 10:01:01,780 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 10:01:01,781 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:01:01,782 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:01:01,787 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:01,910 DEBUG   SenderThread:1511293 [sender.py:send():379] send: telemetry
+2024-12-31 10:01:01,911 DEBUG   SenderThread:1511293 [sender.py:send():379] send: config
+2024-12-31 10:01:01,911 DEBUG   SenderThread:1511293 [sender.py:send():379] send: telemetry
+2024-12-31 10:01:01,911 DEBUG   SenderThread:1511293 [sender.py:send():379] send: metric
+2024-12-31 10:01:01,911 DEBUG   SenderThread:1511293 [sender.py:send():379] send: telemetry
+2024-12-31 10:01:01,911 DEBUG   SenderThread:1511293 [sender.py:send():379] send: metric
+2024-12-31 10:01:01,912 WARNING SenderThread:1511293 [sender.py:send_metric():1405] Seen metric with glob (shouldn't happen)
+2024-12-31 10:01:01,912 DEBUG   SenderThread:1511293 [sender.py:send():379] send: telemetry
+2024-12-31 10:01:01,912 DEBUG   SenderThread:1511293 [sender.py:send():379] send: telemetry
+2024-12-31 10:01:01,912 DEBUG   SenderThread:1511293 [sender.py:send():379] send: config
+2024-12-31 10:01:02,060 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/conda-environment.yaml
+2024-12-31 10:01:02,061 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/requirements.txt
+2024-12-31 10:01:02,061 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/wandb-metadata.json
+2024-12-31 10:01:02,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:03,784 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:04,784 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:05,788 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:05,912 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:06,788 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:07,788 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:08,788 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:09,788 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:10,789 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:11,782 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:11,789 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:12,789 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:13,789 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:14,789 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:15,789 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:16,780 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:01:16,781 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:01:16,821 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:16,948 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:17,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:18,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:19,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:20,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:21,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:22,078 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/output.log
+2024-12-31 10:01:22,782 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:22,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:23,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:24,080 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/output.log
+2024-12-31 10:01:24,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:25,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:26,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:27,788 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:27,796 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:28,083 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/config.yaml
+2024-12-31 10:01:28,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:29,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:30,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:31,780 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:01:31,781 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:01:31,820 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:32,792 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:33,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:33,879 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:34,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:35,792 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:36,791 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:37,792 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:38,792 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:39,782 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:39,792 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:40,792 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:41,794 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:42,797 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:43,802 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:44,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:44,799 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:45,803 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:46,782 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:01:46,783 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:01:46,824 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:47,799 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:48,799 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:49,799 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:49,887 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:50,057 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 10:01:50,058 DEBUG   SenderThread:1511293 [sender.py:send():379] send: metric
+2024-12-31 10:01:50,059 DEBUG   SenderThread:1511293 [sender.py:send():379] send: metric
+2024-12-31 10:01:50,059 DEBUG   SenderThread:1511293 [sender.py:send():379] send: metric
+2024-12-31 10:01:50,059 DEBUG   SenderThread:1511293 [sender.py:send():379] send: metric
+2024-12-31 10:01:50,059 DEBUG   SenderThread:1511293 [sender.py:send():379] send: metric
+2024-12-31 10:01:50,059 DEBUG   SenderThread:1511293 [sender.py:send():379] send: history
+2024-12-31 10:01:50,059 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 10:01:50,064 INFO    SenderThread:1511293 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 10:01:50,110 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_created():271] file/dir created: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/wandb-summary.json
+2024-12-31 10:01:50,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:51,799 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:52,112 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/output.log
+2024-12-31 10:01:52,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:53,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:54,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:55,187 DEBUG   SystemMonitor:1511293 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 10:01:55,190 DEBUG   SenderThread:1511293 [sender.py:send():379] send: stats
+2024-12-31 10:01:55,192 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:01:55,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:56,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:57,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:58,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:01:59,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:00,790 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:00,800 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:01,119 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/config.yaml
+2024-12-31 10:02:01,782 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:02:01,783 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:02:01,823 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:02,801 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:03,801 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:04,965 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:05,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:06,784 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:06,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:07,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:08,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:09,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:10,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:11,784 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:11,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:12,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:13,877 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:14,876 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:15,877 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:16,783 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:02:16,783 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:02:16,877 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:16,903 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:17,877 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:18,887 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:19,880 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:20,880 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:21,882 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:22,792 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:22,882 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:23,883 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:24,883 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:25,192 DEBUG   SenderThread:1511293 [sender.py:send():379] send: stats
+2024-12-31 10:02:25,461 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 10:02:25,463 DEBUG   SenderThread:1511293 [sender.py:send():379] send: history
+2024-12-31 10:02:25,464 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 10:02:25,465 INFO    SenderThread:1511293 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 10:02:25,883 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:26,145 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/wandb-summary.json
+2024-12-31 10:02:26,146 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/output.log
+2024-12-31 10:02:26,883 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:27,883 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:28,788 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:28,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:29,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:30,883 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:31,783 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:02:31,783 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:02:31,883 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:32,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:33,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:33,938 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:34,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:35,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:36,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:37,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:38,884 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:39,787 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:40,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:41,031 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:42,031 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:43,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:44,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:44,787 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:45,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:46,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:46,783 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:02:46,783 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:02:47,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:48,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:49,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:49,855 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:50,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:51,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:52,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:53,032 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:54,046 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:55,042 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:55,206 DEBUG   SenderThread:1511293 [sender.py:send():379] send: stats
+2024-12-31 10:02:55,207 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:02:56,042 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:57,045 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:58,045 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:02:59,045 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:03:00,045 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:03:00,673 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 10:03:00,676 DEBUG   SenderThread:1511293 [sender.py:send():379] send: history
+2024-12-31 10:03:00,677 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 10:03:00,677 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 10:03:00,679 INFO    SenderThread:1511293 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 10:03:01,103 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:03:01,182 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/wandb-summary.json
+2024-12-31 10:03:01,783 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 10:03:01,784 DEBUG   SenderThread:1511293 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 10:03:02,162 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 10:03:02,183 INFO    Thread-12 :1511293 [dir_watcher.py:_on_file_modified():288] file/dir modified: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/files/output.log
+2024-12-31 10:03:03,162 DEBUG   HandlerThread:1511293 [handler.py:handle_request():158] handle_request: internal_messages

wandb/run-20241231_100054-t2idt8o9/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Current SDK version is 0.17.3
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Configure stats pid to 1510803
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Loading settings from /home/ctpham_umass_edu/.config/wandb/settings
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Loading settings from /work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/wandb/settings
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'project': 'prolong', 'api_key': '***REDACTED***', 'mode': 'online'}
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'prolong-final/finetune.py', 'program_abspath': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py'}
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_init.py:_log_setup():520] Logging user logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/logs/debug.log
+2024-12-31 10:00:54,775 INFO    MainThread:1510803 [wandb_init.py:_log_setup():521] Logging internal logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20241231_100054-t2idt8o9/logs/debug-internal.log
+2024-12-31 10:00:54,776 INFO    MainThread:1510803 [wandb_init.py:init():560] calling init triggers
+2024-12-31 10:00:54,776 INFO    MainThread:1510803 [wandb_init.py:init():567] wandb.init called with sweep_config: {}
+config: {}
+2024-12-31 10:00:54,776 INFO    MainThread:1510803 [wandb_init.py:init():610] starting backend
+2024-12-31 10:00:54,776 INFO    MainThread:1510803 [wandb_init.py:init():614] setting up manager
+2024-12-31 10:00:54,778 INFO    MainThread:1510803 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 10:00:54,783 INFO    MainThread:1510803 [wandb_init.py:init():622] backend started and connected
+2024-12-31 10:00:54,788 INFO    MainThread:1510803 [wandb_init.py:init():711] updated telemetry
+2024-12-31 10:00:54,824 INFO    MainThread:1510803 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout
+2024-12-31 10:00:55,080 INFO    MainThread:1510803 [wandb_run.py:_on_init():2402] communicating current version
+2024-12-31 10:00:55,143 INFO    MainThread:1510803 [wandb_run.py:_on_init():2411] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 10:00:55,144 INFO    MainThread:1510803 [wandb_init.py:init():795] starting run threads in backend
+2024-12-31 10:01:01,781 INFO    MainThread:1510803 [wandb_run.py:_console_start():2380] atexit reg
+2024-12-31 10:01:01,781 INFO    MainThread:1510803 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2024-12-31 10:01:01,781 INFO    MainThread:1510803 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2024-12-31 10:01:01,781 INFO    MainThread:1510803 [wandb_run.py:_redirect():2325] Redirects installed.
+2024-12-31 10:01:01,785 INFO    MainThread:1510803 [wandb_init.py:init():838] run started, returning control to user process
+2024-12-31 10:01:01,787 INFO    MainThread:1510803 [wandb_run.py:_config_callback():1382] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 32, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': 0, 'eos_token_id': [128001, 128008, 128009], 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', 'transformers_version': '4.44.2', 'model_type': 'llama', 'output_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'overwrite_output_dir': False, 'do_train': True, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-06, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.05, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/runs/Dec31_10-00-19_gpu020', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1.0, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 200, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 1, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'disable_tqdm': True, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': ['auto_wrap', 'offload'], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'eval_use_gather_object': False, 'min_lr_ratio': 0.1, 'cuda_empty_cache': True, 'streaming_dataset': True, 'seq_parallel_size': 8}
+2024-12-31 10:01:01,790 INFO    MainThread:1510803 [wandb_config.py:__setitem__():151] config set model/num_parameters = 1003782656 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x74daa2385f90>>
+2024-12-31 10:01:01,790 INFO    MainThread:1510803 [wandb_run.py:_config_callback():1382] config_cb model/num_parameters 1003782656 None

wandb/run-20250101_112144-t9wzg2aq/files/conda-environment.yaml

@@ -0,0 +1,233 @@
+name: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - bzip2=1.0.8=h4bc722e_7
+  - ca-certificates=2024.12.14=hbcca054_0
+  - ld_impl_linux-64=2.43=h712a8e2_2
+  - libffi=3.4.2=h7f98852_5
+  - libgcc=14.2.0=h77fa898_1
+  - libgcc-ng=14.2.0=h69a702a_1
+  - libgomp=14.2.0=h77fa898_1
+  - liblzma=5.6.3=hb9d3cd8_1
+  - liblzma-devel=5.6.3=hb9d3cd8_1
+  - libnsl=2.0.1=hd590300_0
+  - libsqlite=3.47.2=hee588c1_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.3.1=hb9d3cd8_2
+  - ncurses=6.5=he02047a_1
+  - openssl=3.4.0=hb9d3cd8_0
+  - pip=24.3.1=pyh8b19718_2
+  - python=3.10.0=h543edf9_3_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=75.6.0=pyhff2d567_1
+  - sqlite=3.47.2=h9eae976_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - wheel=0.45.1=pyhd8ed1ab_1
+  - xz=5.6.3=hbcc6ac9_1
+  - xz-gpl-tools=5.6.3=hbcc6ac9_1
+  - xz-tools=5.6.3=hb9d3cd8_1
+  - pip:
+      - accelerate==0.32.1
+      - aiohappyeyeballs==2.4.3
+      - aiohttp==3.11.2
+      - aioprometheus==23.12.0
+      - aiosignal==1.3.1
+      - annotated-types==0.7.0
+      - anthropic==0.39.0
+      - anyio==4.6.2.post1
+      - argcomplete==3.5.1
+      - arrow==1.3.0
+      - async-timeout==5.0.1
+      - attrs==24.2.0
+      - azure-core==1.32.0
+      - azure-identity==1.19.0
+      - azure-storage-blob==12.24.0
+      - azure-storage-file-datalake==12.18.0
+      - backoff==2.2.1
+      - bcrypt==4.2.0
+      - blobfile==3.0.0
+      - boto3==1.35.63
+      - botocore==1.35.63
+      - brotli==1.1.0
+      - cachetools==5.5.0
+      - certifi==2024.8.30
+      - cffi==1.17.1
+      - charset-normalizer==3.4.0
+      - circuitbreaker==2.0.0
+      - click==8.1.7
+      - cloudpickle==3.1.0
+      - compressed-tensors==0.8.0
+      - contourpy==1.3.1
+      - cramjam==2.9.0
+      - cryptography==43.0.3
+      - cycler==0.12.1
+      - datasets==2.20.0
+      - debugpy==1.8.11
+      - dill==0.3.8
+      - diskcache==5.6.3
+      - distro==1.9.0
+      - docker-pycreds==0.4.0
+      - docstring-parser==0.16
+      - einops==0.8.0
+      - fastapi==0.115.5
+      - filelock==3.16.1
+      - flash-attn==2.6.1
+      - fonttools==4.55.0
+      - frozenlist==1.5.0
+      - fsspec==2024.5.0
+      - gguf==0.10.0
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - google-api-core==2.23.0
+      - google-auth==2.36.0
+      - google-cloud-aiplatform==1.71.1
+      - google-cloud-bigquery==3.27.0
+      - google-cloud-core==2.4.1
+      - google-cloud-resource-manager==1.13.1
+      - google-cloud-storage==2.10.0
+      - google-crc32c==1.6.0
+      - google-resumable-media==2.7.2
+      - googleapis-common-protos==1.66.0
+      - gql==3.5.0
+      - graphql-core==3.2.5
+      - grpc-google-iam-v1==0.13.1
+      - grpcio==1.68.0
+      - grpcio-status==1.62.3
+      - h11==0.14.0
+      - httpcore==1.0.7
+      - httptools==0.6.4
+      - httpx==0.27.2
+      - huggingface-hub==0.26.2
+      - idna==3.10
+      - importlib-metadata==8.5.0
+      - interegular==0.3.3
+      - ipython==8.18.0
+      - isodate==0.7.2
+      - jedi==0.19.2
+      - jinja2==3.1.4
+      - jiter==0.7.1
+      - jmespath==1.0.1
+      - jsonschema==4.23.0
+      - jsonschema-specifications==2024.10.1
+      - kiwisolver==1.4.7
+      - lark==1.2.2
+      - llvmlite==0.43.0
+      - lm-format-enforcer==0.10.9
+      - lxml==5.3.0
+      - markdown-it-py==3.0.0
+      - markupsafe==3.0.2
+      - matplotlib==3.9.2
+      - mdurl==0.1.2
+      - mosaicml-cli==0.5.34
+      - mosaicml-streaming==0.8.1
+      - mpmath==1.3.0
+      - msal==1.31.1
+      - msal-extensions==1.2.0
+      - msgpack==1.1.0
+      - msgspec==0.18.6
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - networkx==3.4.2
+      - ninja==1.11.1.1
+      - numba==0.60.0
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==9.1.0.70
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.4.127
+      - nvidia-nvtx-cu12==12.1.105
+      - oci==2.138.1
+      - openai==1.54.5
+      - opencv-python-headless==4.10.0.84
+      - orjson==3.10.11
+      - outlines==0.0.46
+      - packaging==24.1
+      - pandas==2.2.1
+      - paramiko==3.5.0
+      - partial-json-parser==0.2.1.1.post4
+      - pillow==10.4.0
+      - portalocker==2.10.1
+      - prometheus-client==0.21.0
+      - prometheus-fastapi-instrumentator==7.0.0
+      - prompt-toolkit==3.0.36
+      - propcache==0.2.0
+      - proto-plus==1.25.0
+      - protobuf==4.25.3
+      - py-cpuinfo==9.0.0
+      - pyairports==2.1.1
+      - pyarrow==18.0.0
+      - pyarrow-hotfix==0.6
+      - pyasn1==0.6.1
+      - pyasn1-modules==0.4.1
+      - pycountry==24.6.1
+      - pycparser==2.22
+      - pycryptodomex==3.21.0
+      - pydantic==2.9.2
+      - pydantic-core==2.23.4
+      - pyjwt==2.10.0
+      - pynacl==1.5.0
+      - pyopenssl==24.2.1
+      - pyparsing==3.2.0
+      - python-dateutil==2.9.0
+      - python-dotenv==1.0.1
+      - python-snappy==0.7.3
+      - pytz==2024.2
+      - pyyaml==6.0.2
+      - quantile-python==1.1
+      - questionary==2.0.1
+      - ray==2.39.0
+      - referencing==0.35.1
+      - regex==2023.12.25
+      - requests==2.32.3
+      - rich==13.9.4
+      - rotary-emb==0.5.2
+      - rpds-py==0.21.0
+      - rsa==4.9
+      - ruamel-yaml==0.18.6
+      - ruamel-yaml-clib==0.2.12
+      - s3transfer==0.10.3
+      - safetensors==0.4.5
+      - sentencepiece==0.1.99
+      - sentry-sdk==2.18.0
+      - setproctitle==1.3.4
+      - shapely==2.0.6
+      - simple-parsing==0.1.6
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - starlette==0.41.3
+      - sympy==1.13.1
+      - tiktoken==0.7.0
+      - tokenizers==0.19.1
+      - torch==2.4.1
+      - torchvision==0.19.1
+      - tqdm==4.66.4
+      - transformers==4.44.2
+      - triton==3.0.0
+      - types-python-dateutil==2.9.0.20241003
+      - tzdata==2024.2
+      - urllib3==2.2.3
+      - uvicorn==0.32.0
+      - uvloop==0.21.0
+      - validators==0.34.0
+      - vertexai==1.71.1
+      - wandb==0.17.3
+      - watchfiles==0.24.0
+      - websockets==11.0.3
+      - xformers==0.0.28.post1
+      - xxhash==3.5.0
+      - yarl==1.17.2
+      - zipp==3.21.0
+      - zstandard==0.23.0
+      - zstd==1.5.5.1
+prefix: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final

wandb/run-20250101_112144-t9wzg2aq/files/config.yaml

@@ -0,0 +1,713 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.0
+    cli_version: 0.17.3
+    framework: huggingface
+    huggingface_version: 4.44.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735730504
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 105
+      3:
+      - 7
+      - 13
+      - 19
+      - 23
+      - 66
+      4: 3.10.0
+      5: 0.17.3
+      6: 4.44.2
+      8:
+      - 5
+      9:
+        1: transformers_trainer
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+    - 1: train/loss
+      5: 1
+      6:
+      - 1
+    - 1: train/grad_norm
+      5: 1
+      6:
+      - 1
+    - 1: train/learning_rate
+      5: 1
+      6:
+      - 1
+    - 1: train/epoch
+      5: 1
+      6:
+      - 1
+    - 1: train/num_input_tokens_seen
+      5: 1
+      6:
+      - 1
+vocab_size:
+  desc: null
+  value: 128256
+max_position_embeddings:
+  desc: null
+  value: 131072
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 14336
+num_hidden_layers:
+  desc: null
+  value: 32
+num_attention_heads:
+  desc: null
+  value: 32
+num_key_value_heads:
+  desc: null
+  value: 8
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-05
+pretraining_tp:
+  desc: null
+  value: 1
+use_cache:
+  desc: null
+  value: true
+rope_theta:
+  desc: null
+  value: 500000.0
+rope_scaling:
+  desc: null
+  value:
+    factor: 8.0
+    low_freq_factor: 1.0
+    high_freq_factor: 4.0
+    original_max_position_embeddings: 8192
+    rope_type: llama3
+attention_bias:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.0
+mlp_bias:
+  desc: null
+  value: false
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 128000
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value:
+  - 128001
+  - 128008
+  - 128009
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: /datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/
+transformers_version:
+  desc: null
+  value: 4.44.2
+model_type:
+  desc: null
+  value: llama
+output_dir:
+  desc: null
+  value: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: true
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+eval_strategy:
+  desc: null
+  value: 'no'
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 1
+per_device_eval_batch_size:
+  desc: null
+  value: 8
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+gradient_accumulation_steps:
+  desc: null
+  value: 2
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_delay:
+  desc: null
+  value: 0
+torch_empty_cache_steps:
+  desc: null
+  value: null
+learning_rate:
+  desc: null
+  value: 1.0e-06
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 1.0
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+lr_scheduler_kwargs:
+  desc: null
+  value: {}
+warmup_ratio:
+  desc: null
+  value: 0.05
+warmup_steps:
+  desc: null
+  value: 0
+log_level:
+  desc: null
+  value: info
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/runs/Jan01_11-21-09_gpu017
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 1.0
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: steps
+save_steps:
+  desc: null
+  value: 200
+save_total_limit:
+  desc: null
+  value: null
+save_safetensors:
+  desc: null
+  value: true
+save_on_each_node:
+  desc: null
+  value: false
+save_only_model:
+  desc: null
+  value: false
+restore_callback_states_from_checkpoint:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_cpu:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: null
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: null
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: null
+tpu_num_cores:
+  desc: null
+  value: null
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: []
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: null
+dataloader_num_workers:
+  desc: null
+  value: 1
+dataloader_prefetch_factor:
+  desc: null
+  value: null
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: _llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_
+disable_tqdm:
+  desc: null
+  value: true
+remove_unused_columns:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: null
+greater_is_better:
+  desc: null
+  value: null
+ignore_data_skip:
+  desc: null
+  value: false
+fsdp:
+  desc: null
+  value:
+  - auto_wrap
+  - offload
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value:
+    min_num_params: 0
+    xla: false
+    xla_fsdp_v2: false
+    xla_fsdp_grad_ckpt: false
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: null
+accelerator_config:
+  desc: null
+  value:
+    split_batches: false
+    dispatch_batches: null
+    even_batches: true
+    use_seedable_sampler: true
+    non_blocking: false
+    gradient_accumulation_kwargs: null
+deepspeed:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: null
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value:
+  - wandb
+ddp_find_unused_parameters:
+  desc: null
+  value: false
+ddp_bucket_cap_mb:
+  desc: null
+  value: null
+ddp_broadcast_buffers:
+  desc: null
+  value: null
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataloader_persistent_workers:
+  desc: null
+  value: false
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: false
+resume_from_checkpoint:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: null
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+hub_always_push:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+gradient_checkpointing_kwargs:
+  desc: null
+  value: null
+include_inputs_for_metrics:
+  desc: null
+  value: false
+eval_do_concat_batches:
+  desc: null
+  value: true
+fp16_backend:
+  desc: null
+  value: auto
+evaluation_strategy:
+  desc: null
+  value: null
+push_to_hub_model_id:
+  desc: null
+  value: null
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: null
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: null
+torch_compile_mode:
+  desc: null
+  value: null
+dispatch_batches:
+  desc: null
+  value: null
+split_batches:
+  desc: null
+  value: null
+include_tokens_per_second:
+  desc: null
+  value: false
+include_num_input_tokens_seen:
+  desc: null
+  value: false
+neftune_noise_alpha:
+  desc: null
+  value: null
+optim_target_modules:
+  desc: null
+  value: null
+batch_eval_metrics:
+  desc: null
+  value: false
+eval_on_start:
+  desc: null
+  value: false
+eval_use_gather_object:
+  desc: null
+  value: false
+min_lr_ratio:
+  desc: null
+  value: 0.1
+cuda_empty_cache:
+  desc: null
+  value: true
+streaming_dataset:
+  desc: null
+  value: true
+seq_parallel_size:
+  desc: null
+  value: 8
+model/num_parameters:
+  desc: null
+  value: 1003782656

wandb/run-20250101_112144-t9wzg2aq/files/requirements.txt

@@ -0,0 +1,244 @@
+Brotli==1.1.0
+GitPython==3.1.43
+Jinja2==3.1.4
+MarkupSafe==3.0.2
+PyJWT==2.10.0
+PyNaCl==1.5.0
+PyYAML==6.0.2
+Pygments==2.18.0
+accelerate==0.32.1
+aiohappyeyeballs==2.4.3
+aiohttp==3.11.2
+aioprometheus==23.12.0
+aiosignal==1.3.1
+annotated-types==0.7.0
+anthropic==0.39.0
+anyio==4.6.2.post1
+argcomplete==3.5.1
+arrow==1.3.0
+asttokens==2.4.1
+async-timeout==5.0.1
+attrs==24.2.0
+autocommand==2.2.2
+azure-core==1.32.0
+azure-identity==1.19.0
+azure-storage-blob==12.24.0
+azure-storage-file-datalake==12.18.0
+backoff==2.2.1
+backports.tarfile==1.2.0
+bcrypt==4.2.0
+blobfile==3.0.0
+boto3==1.35.63
+botocore==1.35.63
+cachetools==5.5.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.4.0
+circuitbreaker==2.0.0
+click==8.1.7
+cloudpickle==3.1.0
+comm==0.2.2
+compressed-tensors==0.8.0
+contourpy==1.3.1
+cramjam==2.9.0
+cryptography==43.0.3
+cycler==0.12.1
+datasets==2.20.0
+datatools==0.1
+debugpy==1.8.11
+decorator==5.1.1
+dill==0.3.8
+diskcache==5.6.3
+distro==1.9.0
+docker-pycreds==0.4.0
+docstring_parser==0.16
+einops==0.8.0
+exceptiongroup==1.2.2
+executing==2.1.0
+fastapi==0.115.5
+filelock==3.16.1
+flash-attn==2.6.1
+fonttools==4.55.0
+frozenlist==1.5.0
+fsspec==2024.5.0
+gguf==0.10.0
+gitdb==4.0.11
+google-api-core==2.23.0
+google-auth==2.36.0
+google-cloud-aiplatform==1.71.1
+google-cloud-bigquery==3.27.0
+google-cloud-core==2.4.1
+google-cloud-resource-manager==1.13.1
+google-cloud-storage==2.10.0
+google-crc32c==1.6.0
+google-resumable-media==2.7.2
+googleapis-common-protos==1.66.0
+gql==3.5.0
+graphql-core==3.2.5
+grpc-google-iam-v1==0.13.1
+grpcio-status==1.62.3
+grpcio==1.68.0
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.27.2
+huggingface-hub==0.26.2
+idna==3.10
+importlib_metadata==8.0.0
+importlib_metadata==8.5.0
+inflect==7.3.1
+interegular==0.3.3
+ipykernel==6.29.5
+ipython==8.18.0
+isodate==0.7.2
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jedi==0.19.2
+jiter==0.7.1
+jmespath==1.0.1
+jsonschema-specifications==2024.10.1
+jsonschema==4.23.0
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+kiwisolver==1.4.7
+lark==1.2.2
+llvmlite==0.43.0
+lm-format-enforcer==0.10.9
+lxml==5.3.0
+markdown-it-py==3.0.0
+matplotlib-inline==0.1.7
+matplotlib==3.9.2
+mdurl==0.1.2
+more-itertools==10.3.0
+mosaicml-cli==0.5.34
+mosaicml-streaming==0.8.1
+mpmath==1.3.0
+msal-extensions==1.2.0
+msal==1.31.1
+msgpack==1.1.0
+msgspec==0.18.6
+multidict==6.1.0
+multiprocess==0.70.16
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.11.1.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py==12.560.30
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+oci==2.138.1
+openai==1.54.5
+opencv-python-headless==4.10.0.84
+orjson==3.10.11
+outlines==0.0.46
+packaging==24.1
+packaging==24.2
+pandas==2.2.1
+paramiko==3.5.0
+parso==0.8.4
+partial-json-parser==0.2.1.1.post4
+pexpect==4.9.0
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+platformdirs==4.3.6
+portalocker==2.10.1
+prometheus-fastapi-instrumentator==7.0.0
+prometheus_client==0.21.0
+prompt-toolkit==3.0.36
+propcache==0.2.0
+proto-plus==1.25.0
+protobuf==4.25.3
+psutil==6.1.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py-cpuinfo==9.0.0
+pyOpenSSL==24.2.1
+pyairports==2.1.1
+pyarrow-hotfix==0.6
+pyarrow==18.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pycountry==24.6.1
+pycparser==2.22
+pycryptodomex==3.21.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+pyparsing==3.2.0
+python-dateutil==2.9.0
+python-dotenv==1.0.1
+python-snappy==0.7.3
+pytz==2024.2
+pyzmq==26.2.0
+quantile-python==1.1
+questionary==2.0.1
+ray==2.39.0
+referencing==0.35.1
+regex==2023.12.25
+requests==2.32.3
+rich==13.9.4
+rotary-emb==0.5.2
+rpds-py==0.21.0
+rsa==4.9
+ruamel.yaml.clib==0.2.12
+ruamel.yaml==0.18.6
+s3transfer==0.10.3
+safetensors==0.4.5
+sentencepiece==0.1.99
+sentry-sdk==2.18.0
+setproctitle==1.3.4
+setuptools==75.6.0
+shapely==2.0.6
+simple-parsing==0.1.6
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+stack-data==0.6.3
+starlette==0.41.3
+sympy==1.13.1
+tiktoken==0.7.0
+tokenizers==0.19.1
+tomli==2.0.1
+torch==2.4.1
+torchvision==0.19.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.44.2
+triton==3.0.0
+typeguard==4.3.0
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+typing_extensions==4.12.2
+tzdata==2024.2
+urllib3==2.2.3
+uvicorn==0.32.0
+uvloop==0.21.0
+validators==0.34.0
+vertexai==1.71.1
+wandb==0.17.3
+watchfiles==0.24.0
+wcwidth==0.2.13
+websockets==11.0.3
+wheel==0.43.0
+wheel==0.45.1
+xformers==0.0.28.post1
+xxhash==3.5.0
+yarl==1.17.2
+zipp==3.19.2
+zipp==3.21.0
+zstandard==0.23.0
+zstd==1.5.5.1

wandb/run-20250101_112144-t9wzg2aq/files/wandb-metadata.json

@@ -0,0 +1,705 @@
+{
+    "os": "Linux-6.8.0-48-generic-x86_64-with-glibc2.39",
+    "python": "3.10.0",
+    "heartbeatAt": "2025-01-01T11:21:45.060977",
+    "startedAt": "2025-01-01T11:21:44.520147",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_family",
+        "llama",
+        "--apply_instruct_masks",
+        "--token_scaled_loss",
+        "--seq_parallel_size",
+        "8",
+        "--report_to",
+        "wandb",
+        "--do_train",
+        "--model_name_or_path",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--config_name",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--tokenizer_name",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--run_name",
+        "_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_",
+        "--output_dir",
+        "/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_",
+        "--config_overrides_json",
+        "",
+        "--gradient_accumulation_steps",
+        "2",
+        "--per_device_train_batch_size",
+        "1",
+        "--bf16",
+        "--learning_rate",
+        "1e-6",
+        "--min_lr_ratio",
+        "0.1",
+        "--lr_scheduler_type",
+        "cosine",
+        "--max_grad_norm",
+        "1.0",
+        "--adam_beta1",
+        "0.9",
+        "--adam_beta2",
+        "0.95",
+        "--weight_decay",
+        "0.1",
+        "--warmup_ratio",
+        "0.05",
+        "--optim",
+        "adamw_torch",
+        "--logging_steps",
+        "1",
+        "--log_level",
+        "info",
+        "--save_steps",
+        "200",
+        "--dataloader_num_workers",
+        "1",
+        "--disable_tqdm",
+        "true",
+        "--use_fast_tokenizer",
+        "false",
+        "--remove_unused_columns",
+        "false",
+        "--ddp_find_unused_parameters",
+        "false",
+        "--fsdp",
+        "auto_wrap offload",
+        "--gradient_checkpointing",
+        "--tokenized_mds_train",
+        "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete",
+        "--cuda_empty_cache",
+        "--num_train_epochs",
+        "1"
+    ],
+    "state": "running",
+    "program": "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py",
+    "codePathLocal": "finetune.py",
+    "codePath": "prolong-final/finetune.py",
+    "git": {
+        "remote": "https://github.com/chtmp223/BookGen-dev.git",
+        "commit": "0e796521430a0f767be7c4dadba5c2fcaee1f909"
+    },
+    "email": "[email protected]",
+    "root": "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev",
+    "host": "gpu017",
+    "username": "ctpham_umass_edu",
+    "executable": "/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10",
+    "cpu_count": 112,
+    "cpu_count_logical": 112,
+    "cpu_freq": {
+        "current": 975.6290892857141,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 762.481,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2100.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 785.34,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1300.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1103.96,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 790.077,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1587.993,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.494,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2547.505,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2400.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2400.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.425,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1997.889,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1300.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 782.826,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1100.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 768.56,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 783.299,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 782.047,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 794.129,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 438.487850189209,
+            "used": 16.457809448242188
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-80GB",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        }
+    ],
+    "memory": {
+        "total": 2015.3286399841309
+    }
+}

wandb/run-20250101_112144-t9wzg2aq/logs/debug.log

@@ -0,0 +1,31 @@
+2025-01-01 11:21:44,529 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Current SDK version is 0.17.3
+2025-01-01 11:21:44,529 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Configure stats pid to 2689681
+2025-01-01 11:21:44,529 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Loading settings from /home/ctpham_umass_edu/.config/wandb/settings
+2025-01-01 11:21:44,529 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Loading settings from /work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/wandb/settings
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'project': 'prolong', 'api_key': '***REDACTED***', 'mode': 'online'}
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'prolong-final/finetune.py', 'program_abspath': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py'}
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_init.py:_log_setup():520] Logging user logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/logs/debug.log
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_init.py:_log_setup():521] Logging internal logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250101_112144-t9wzg2aq/logs/debug-internal.log
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_init.py:init():560] calling init triggers
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_init.py:init():567] wandb.init called with sweep_config: {}
+config: {}
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_init.py:init():610] starting backend
+2025-01-01 11:21:44,530 INFO    MainThread:2689681 [wandb_init.py:init():614] setting up manager
+2025-01-01 11:21:44,533 INFO    MainThread:2689681 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-01 11:21:44,536 INFO    MainThread:2689681 [wandb_init.py:init():622] backend started and connected
+2025-01-01 11:21:44,544 INFO    MainThread:2689681 [wandb_init.py:init():711] updated telemetry
+2025-01-01 11:21:44,575 INFO    MainThread:2689681 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout
+2025-01-01 11:21:44,902 INFO    MainThread:2689681 [wandb_run.py:_on_init():2402] communicating current version
+2025-01-01 11:21:44,965 INFO    MainThread:2689681 [wandb_run.py:_on_init():2411] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-01 11:21:44,965 INFO    MainThread:2689681 [wandb_init.py:init():795] starting run threads in backend
+2025-01-01 11:21:51,404 INFO    MainThread:2689681 [wandb_run.py:_console_start():2380] atexit reg
+2025-01-01 11:21:51,404 INFO    MainThread:2689681 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2025-01-01 11:21:51,404 INFO    MainThread:2689681 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2025-01-01 11:21:51,404 INFO    MainThread:2689681 [wandb_run.py:_redirect():2325] Redirects installed.
+2025-01-01 11:21:51,407 INFO    MainThread:2689681 [wandb_init.py:init():838] run started, returning control to user process
+2025-01-01 11:21:51,408 INFO    MainThread:2689681 [wandb_run.py:_config_callback():1382] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 32, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': 0, 'eos_token_id': [128001, 128008, 128009], 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', 'transformers_version': '4.44.2', 'model_type': 'llama', 'output_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'overwrite_output_dir': False, 'do_train': True, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-06, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.05, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/runs/Jan01_11-21-09_gpu017', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1.0, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 200, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 1, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'disable_tqdm': True, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': ['auto_wrap', 'offload'], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'eval_use_gather_object': False, 'min_lr_ratio': 0.1, 'cuda_empty_cache': True, 'streaming_dataset': True, 'seq_parallel_size': 8}
+2025-01-01 11:21:51,411 INFO    MainThread:2689681 [wandb_config.py:__setitem__():151] config set model/num_parameters = 1003782656 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x73716eb5e260>>
+2025-01-01 11:21:51,411 INFO    MainThread:2689681 [wandb_run.py:_config_callback():1382] config_cb model/num_parameters 1003782656 None

wandb/run-20250101_112144-t9wzg2aq/run-t9wzg2aq.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24d095c55a3634e35a51c48500ba917124bdc8e697787fb2642a5f1f31ace3f3
+size 6852892

wandb/run-20250102_021927-pw8rud5e/files/conda-environment.yaml

@@ -0,0 +1,233 @@
+name: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - bzip2=1.0.8=h4bc722e_7
+  - ca-certificates=2024.12.14=hbcca054_0
+  - ld_impl_linux-64=2.43=h712a8e2_2
+  - libffi=3.4.2=h7f98852_5
+  - libgcc=14.2.0=h77fa898_1
+  - libgcc-ng=14.2.0=h69a702a_1
+  - libgomp=14.2.0=h77fa898_1
+  - liblzma=5.6.3=hb9d3cd8_1
+  - liblzma-devel=5.6.3=hb9d3cd8_1
+  - libnsl=2.0.1=hd590300_0
+  - libsqlite=3.47.2=hee588c1_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.3.1=hb9d3cd8_2
+  - ncurses=6.5=he02047a_1
+  - openssl=3.4.0=hb9d3cd8_0
+  - pip=24.3.1=pyh8b19718_2
+  - python=3.10.0=h543edf9_3_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=75.6.0=pyhff2d567_1
+  - sqlite=3.47.2=h9eae976_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - wheel=0.45.1=pyhd8ed1ab_1
+  - xz=5.6.3=hbcc6ac9_1
+  - xz-gpl-tools=5.6.3=hbcc6ac9_1
+  - xz-tools=5.6.3=hb9d3cd8_1
+  - pip:
+      - accelerate==0.32.1
+      - aiohappyeyeballs==2.4.3
+      - aiohttp==3.11.2
+      - aioprometheus==23.12.0
+      - aiosignal==1.3.1
+      - annotated-types==0.7.0
+      - anthropic==0.39.0
+      - anyio==4.6.2.post1
+      - argcomplete==3.5.1
+      - arrow==1.3.0
+      - async-timeout==5.0.1
+      - attrs==24.2.0
+      - azure-core==1.32.0
+      - azure-identity==1.19.0
+      - azure-storage-blob==12.24.0
+      - azure-storage-file-datalake==12.18.0
+      - backoff==2.2.1
+      - bcrypt==4.2.0
+      - blobfile==3.0.0
+      - boto3==1.35.63
+      - botocore==1.35.63
+      - brotli==1.1.0
+      - cachetools==5.5.0
+      - certifi==2024.8.30
+      - cffi==1.17.1
+      - charset-normalizer==3.4.0
+      - circuitbreaker==2.0.0
+      - click==8.1.7
+      - cloudpickle==3.1.0
+      - compressed-tensors==0.8.0
+      - contourpy==1.3.1
+      - cramjam==2.9.0
+      - cryptography==43.0.3
+      - cycler==0.12.1
+      - datasets==2.20.0
+      - debugpy==1.8.11
+      - dill==0.3.8
+      - diskcache==5.6.3
+      - distro==1.9.0
+      - docker-pycreds==0.4.0
+      - docstring-parser==0.16
+      - einops==0.8.0
+      - fastapi==0.115.5
+      - filelock==3.16.1
+      - flash-attn==2.6.1
+      - fonttools==4.55.0
+      - frozenlist==1.5.0
+      - fsspec==2024.5.0
+      - gguf==0.10.0
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - google-api-core==2.23.0
+      - google-auth==2.36.0
+      - google-cloud-aiplatform==1.71.1
+      - google-cloud-bigquery==3.27.0
+      - google-cloud-core==2.4.1
+      - google-cloud-resource-manager==1.13.1
+      - google-cloud-storage==2.10.0
+      - google-crc32c==1.6.0
+      - google-resumable-media==2.7.2
+      - googleapis-common-protos==1.66.0
+      - gql==3.5.0
+      - graphql-core==3.2.5
+      - grpc-google-iam-v1==0.13.1
+      - grpcio==1.68.0
+      - grpcio-status==1.62.3
+      - h11==0.14.0
+      - httpcore==1.0.7
+      - httptools==0.6.4
+      - httpx==0.27.2
+      - huggingface-hub==0.26.2
+      - idna==3.10
+      - importlib-metadata==8.5.0
+      - interegular==0.3.3
+      - ipython==8.18.0
+      - isodate==0.7.2
+      - jedi==0.19.2
+      - jinja2==3.1.4
+      - jiter==0.7.1
+      - jmespath==1.0.1
+      - jsonschema==4.23.0
+      - jsonschema-specifications==2024.10.1
+      - kiwisolver==1.4.7
+      - lark==1.2.2
+      - llvmlite==0.43.0
+      - lm-format-enforcer==0.10.9
+      - lxml==5.3.0
+      - markdown-it-py==3.0.0
+      - markupsafe==3.0.2
+      - matplotlib==3.9.2
+      - mdurl==0.1.2
+      - mosaicml-cli==0.5.34
+      - mosaicml-streaming==0.8.1
+      - mpmath==1.3.0
+      - msal==1.31.1
+      - msal-extensions==1.2.0
+      - msgpack==1.1.0
+      - msgspec==0.18.6
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - networkx==3.4.2
+      - ninja==1.11.1.1
+      - numba==0.60.0
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==9.1.0.70
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.4.127
+      - nvidia-nvtx-cu12==12.1.105
+      - oci==2.138.1
+      - openai==1.54.5
+      - opencv-python-headless==4.10.0.84
+      - orjson==3.10.11
+      - outlines==0.0.46
+      - packaging==24.1
+      - pandas==2.2.1
+      - paramiko==3.5.0
+      - partial-json-parser==0.2.1.1.post4
+      - pillow==10.4.0
+      - portalocker==2.10.1
+      - prometheus-client==0.21.0
+      - prometheus-fastapi-instrumentator==7.0.0
+      - prompt-toolkit==3.0.36
+      - propcache==0.2.0
+      - proto-plus==1.25.0
+      - protobuf==4.25.3
+      - py-cpuinfo==9.0.0
+      - pyairports==2.1.1
+      - pyarrow==18.0.0
+      - pyarrow-hotfix==0.6
+      - pyasn1==0.6.1
+      - pyasn1-modules==0.4.1
+      - pycountry==24.6.1
+      - pycparser==2.22
+      - pycryptodomex==3.21.0
+      - pydantic==2.9.2
+      - pydantic-core==2.23.4
+      - pyjwt==2.10.0
+      - pynacl==1.5.0
+      - pyopenssl==24.2.1
+      - pyparsing==3.2.0
+      - python-dateutil==2.9.0
+      - python-dotenv==1.0.1
+      - python-snappy==0.7.3
+      - pytz==2024.2
+      - pyyaml==6.0.2
+      - quantile-python==1.1
+      - questionary==2.0.1
+      - ray==2.39.0
+      - referencing==0.35.1
+      - regex==2023.12.25
+      - requests==2.32.3
+      - rich==13.9.4
+      - rotary-emb==0.5.2
+      - rpds-py==0.21.0
+      - rsa==4.9
+      - ruamel-yaml==0.18.6
+      - ruamel-yaml-clib==0.2.12
+      - s3transfer==0.10.3
+      - safetensors==0.4.5
+      - sentencepiece==0.1.99
+      - sentry-sdk==2.18.0
+      - setproctitle==1.3.4
+      - shapely==2.0.6
+      - simple-parsing==0.1.6
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - starlette==0.41.3
+      - sympy==1.13.1
+      - tiktoken==0.7.0
+      - tokenizers==0.19.1
+      - torch==2.4.1
+      - torchvision==0.19.1
+      - tqdm==4.66.4
+      - transformers==4.44.2
+      - triton==3.0.0
+      - types-python-dateutil==2.9.0.20241003
+      - tzdata==2024.2
+      - urllib3==2.2.3
+      - uvicorn==0.32.0
+      - uvloop==0.21.0
+      - validators==0.34.0
+      - vertexai==1.71.1
+      - wandb==0.17.3
+      - watchfiles==0.24.0
+      - websockets==11.0.3
+      - xformers==0.0.28.post1
+      - xxhash==3.5.0
+      - yarl==1.17.2
+      - zipp==3.21.0
+      - zstandard==0.23.0
+      - zstd==1.5.5.1
+prefix: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final

wandb/run-20250102_021927-pw8rud5e/files/config.yaml

@@ -0,0 +1,713 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.0
+    cli_version: 0.17.3
+    framework: huggingface
+    huggingface_version: 4.44.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735784367
+    t:
+      1:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 41
+      - 49
+      - 51
+      - 55
+      - 71
+      - 105
+      3:
+      - 7
+      - 13
+      - 19
+      - 23
+      - 66
+      4: 3.10.0
+      5: 0.17.3
+      6: 4.44.2
+      8:
+      - 5
+      9:
+        1: transformers_trainer
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+    - 1: train/loss
+      5: 1
+      6:
+      - 1
+    - 1: train/grad_norm
+      5: 1
+      6:
+      - 1
+    - 1: train/learning_rate
+      5: 1
+      6:
+      - 1
+    - 1: train/epoch
+      5: 1
+      6:
+      - 1
+    - 1: train/num_input_tokens_seen
+      5: 1
+      6:
+      - 1
+vocab_size:
+  desc: null
+  value: 128256
+max_position_embeddings:
+  desc: null
+  value: 131072
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 14336
+num_hidden_layers:
+  desc: null
+  value: 32
+num_attention_heads:
+  desc: null
+  value: 32
+num_key_value_heads:
+  desc: null
+  value: 8
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-05
+pretraining_tp:
+  desc: null
+  value: 1
+use_cache:
+  desc: null
+  value: true
+rope_theta:
+  desc: null
+  value: 500000.0
+rope_scaling:
+  desc: null
+  value:
+    factor: 8.0
+    low_freq_factor: 1.0
+    high_freq_factor: 4.0
+    original_max_position_embeddings: 8192
+    rope_type: llama3
+attention_bias:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.0
+mlp_bias:
+  desc: null
+  value: false
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 128000
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value:
+  - 128001
+  - 128008
+  - 128009
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: /datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/
+transformers_version:
+  desc: null
+  value: 4.44.2
+model_type:
+  desc: null
+  value: llama
+output_dir:
+  desc: null
+  value: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: true
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+eval_strategy:
+  desc: null
+  value: 'no'
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 1
+per_device_eval_batch_size:
+  desc: null
+  value: 8
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+gradient_accumulation_steps:
+  desc: null
+  value: 2
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_delay:
+  desc: null
+  value: 0
+torch_empty_cache_steps:
+  desc: null
+  value: null
+learning_rate:
+  desc: null
+  value: 1.0e-06
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 1.0
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+lr_scheduler_kwargs:
+  desc: null
+  value: {}
+warmup_ratio:
+  desc: null
+  value: 0.05
+warmup_steps:
+  desc: null
+  value: 0
+log_level:
+  desc: null
+  value: info
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/runs/Jan02_02-12-22_gpu020
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 1.0
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: steps
+save_steps:
+  desc: null
+  value: 200
+save_total_limit:
+  desc: null
+  value: null
+save_safetensors:
+  desc: null
+  value: true
+save_on_each_node:
+  desc: null
+  value: false
+save_only_model:
+  desc: null
+  value: false
+restore_callback_states_from_checkpoint:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_cpu:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: null
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: null
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: null
+tpu_num_cores:
+  desc: null
+  value: null
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: []
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: null
+dataloader_num_workers:
+  desc: null
+  value: 1
+dataloader_prefetch_factor:
+  desc: null
+  value: null
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: _llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_
+disable_tqdm:
+  desc: null
+  value: true
+remove_unused_columns:
+  desc: null
+  value: false
+label_names:
+  desc: null
+  value: null
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: null
+greater_is_better:
+  desc: null
+  value: null
+ignore_data_skip:
+  desc: null
+  value: true
+fsdp:
+  desc: null
+  value:
+  - auto_wrap
+  - offload
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value:
+    min_num_params: 0
+    xla: false
+    xla_fsdp_v2: false
+    xla_fsdp_grad_ckpt: false
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: null
+accelerator_config:
+  desc: null
+  value:
+    split_batches: false
+    dispatch_batches: null
+    even_batches: true
+    use_seedable_sampler: true
+    non_blocking: false
+    gradient_accumulation_kwargs: null
+deepspeed:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: null
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value:
+  - wandb
+ddp_find_unused_parameters:
+  desc: null
+  value: false
+ddp_bucket_cap_mb:
+  desc: null
+  value: null
+ddp_broadcast_buffers:
+  desc: null
+  value: null
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataloader_persistent_workers:
+  desc: null
+  value: false
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: false
+resume_from_checkpoint:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: null
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+hub_always_push:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+gradient_checkpointing_kwargs:
+  desc: null
+  value: null
+include_inputs_for_metrics:
+  desc: null
+  value: false
+eval_do_concat_batches:
+  desc: null
+  value: true
+fp16_backend:
+  desc: null
+  value: auto
+evaluation_strategy:
+  desc: null
+  value: null
+push_to_hub_model_id:
+  desc: null
+  value: null
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: null
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: null
+torch_compile_mode:
+  desc: null
+  value: null
+dispatch_batches:
+  desc: null
+  value: null
+split_batches:
+  desc: null
+  value: null
+include_tokens_per_second:
+  desc: null
+  value: false
+include_num_input_tokens_seen:
+  desc: null
+  value: false
+neftune_noise_alpha:
+  desc: null
+  value: null
+optim_target_modules:
+  desc: null
+  value: null
+batch_eval_metrics:
+  desc: null
+  value: false
+eval_on_start:
+  desc: null
+  value: false
+eval_use_gather_object:
+  desc: null
+  value: false
+min_lr_ratio:
+  desc: null
+  value: 0.1
+cuda_empty_cache:
+  desc: null
+  value: true
+streaming_dataset:
+  desc: null
+  value: true
+seq_parallel_size:
+  desc: null
+  value: 8
+model/num_parameters:
+  desc: null
+  value: 1003782656

wandb/run-20250102_021927-pw8rud5e/files/output.log

@@ -0,0 +1,293 @@
+/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/transformers/trainer.py:2833: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  checkpoint_rng_state = torch.load(rng_file)
+/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context:  # type: ignore[attr-defined]
+/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context:  # type: ignore[attr-defined]
+[INFO|trainer.py:175] 2025-01-02 02:20:19,436 >> {'loss': 0.5452, 'grad_norm': 21.211387634277344, 'learning_rate': 8.847211348554382e-07, 'epoch': 0.00027122321670735016, 'num_input_tokens_seen': 2099249152, 'completed': '27.15% (1_001 / 3_687)', 'remaining time': '32:49:21', 'throughput': '2979.49', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:20:50,985 >> {'loss': 0.494, 'grad_norm': 15.134449005126953, 'learning_rate': 8.844511851055991e-07, 'epoch': 0.0005424464334147003, 'num_input_tokens_seen': 2101346304, 'completed': '27.18% (1_002 / 3_687)', 'remaining time': '28:10:14', 'throughput': '8308.84', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:21:21,666 >> {'loss': 0.5585, 'grad_norm': 15.855742454528809, 'learning_rate': 8.841809662021731e-07, 'epoch': 0.0008136696501220504, 'num_input_tokens_seen': 2103443456, 'completed': '27.20% (1_003 / 3_687)', 'remaining time': '26:23:53', 'throughput': '8544.34', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:21:52,330 >> {'loss': 0.9896, 'grad_norm': 25.325538635253906, 'learning_rate': 8.839104783626219e-07, 'epoch': 0.0010848928668294006, 'num_input_tokens_seen': 2105540608, 'completed': '27.23% (1_004 / 3_687)', 'remaining time': '25:30:16', 'throughput': '8548.91', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:22:22,067 >> {'loss': 0.423, 'grad_norm': 12.955252647399902, 'learning_rate': 8.836397218046239e-07, 'epoch': 0.0013561160835367507, 'num_input_tokens_seen': 2107637760, 'completed': '27.26% (1_005 / 3_687)', 'remaining time': '24:49:36', 'throughput': '8815.37', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:22:50,884 >> {'loss': 0.5536, 'grad_norm': 18.2934513092041, 'learning_rate': 8.83368696746074e-07, 'epoch': 0.0016273393002441008, 'num_input_tokens_seen': 2109734912, 'completed': '27.29% (1_006 / 3_687)', 'remaining time': '24:15:29', 'throughput': '9096.90', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:23:19,708 >> {'loss': 0.8425, 'grad_norm': 19.734365463256836, 'learning_rate': 8.830974034050824e-07, 'epoch': 0.001898562516951451, 'num_input_tokens_seen': 2111832064, 'completed': '27.31% (1_007 / 3_687)', 'remaining time': '23:51:00', 'throughput': '9094.73', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:23:48,416 >> {'loss': 0.2779, 'grad_norm': 9.506952285766602, 'learning_rate': 8.828258419999759e-07, 'epoch': 0.0021697857336588013, 'num_input_tokens_seen': 2113929216, 'completed': '27.34% (1_008 / 3_687)', 'remaining time': '23:31:53', 'throughput': '9131.29', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:24:20,107 >> {'loss': 0.3105, 'grad_norm': 10.620465278625488, 'learning_rate': 8.825540127492965e-07, 'epoch': 0.0024410089503661514, 'num_input_tokens_seen': 2116026368, 'completed': '27.37% (1_009 / 3_687)', 'remaining time': '23:31:43', 'throughput': '8271.85', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:24:51,346 >> {'loss': 0.775, 'grad_norm': 20.96041488647461, 'learning_rate': 8.822819158718026e-07, 'epoch': 0.0027122321670735015, 'num_input_tokens_seen': 2118123520, 'completed': '27.39% (1_010 / 3_687)', 'remaining time': '23:29:26', 'throughput': '8391.71', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:25:21,486 >> {'loss': 0.505, 'grad_norm': 15.075592994689941, 'learning_rate': 8.820095515864669e-07, 'epoch': 0.0029834553837808516, 'num_input_tokens_seen': 2120220672, 'completed': '27.42% (1_011 / 3_687)', 'remaining time': '23:23:02', 'throughput': '8697.53', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:25:51,490 >> {'loss': 0.6136, 'grad_norm': 19.47187042236328, 'learning_rate': 8.81736920112478e-07, 'epoch': 0.0032546786004882017, 'num_input_tokens_seen': 2122317824, 'completed': '27.45% (1_012 / 3_687)', 'remaining time': '23:17:07', 'throughput': '8736.74', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:26:26,199 >> {'loss': 0.4918, 'grad_norm': 17.397031784057617, 'learning_rate': 8.814640216692391e-07, 'epoch': 0.003525901817195552, 'num_input_tokens_seen': 2124414976, 'completed': '27.47% (1_013 / 3_687)', 'remaining time': '23:28:09', 'throughput': '7552.64', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:26:53,599 >> {'loss': 0.8976, 'grad_norm': 23.4683895111084, 'learning_rate': 8.81190856476369e-07, 'epoch': 0.003797125033902902, 'num_input_tokens_seen': 2126512128, 'completed': '27.50% (1_014 / 3_687)', 'remaining time': '23:14:16', 'throughput': '9567.54', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:27:22,811 >> {'loss': 0.6695, 'grad_norm': 21.26357078552246, 'learning_rate': 8.809174247537003e-07, 'epoch': 0.0040683482506102524, 'num_input_tokens_seen': 2128609280, 'completed': '27.53% (1_015 / 3_687)', 'remaining time': '23:07:33', 'throughput': '8973.69', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:27:54,975 >> {'loss': 0.2329, 'grad_norm': 9.867420196533203, 'learning_rate': 8.806437267212805e-07, 'epoch': 0.0043395714673176026, 'num_input_tokens_seen': 2130706432, 'completed': '27.56% (1_016 / 3_687)', 'remaining time': '23:09:50', 'throughput': '8150.26', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:28:28,662 >> {'loss': 0.8958, 'grad_norm': 26.58327293395996, 'learning_rate': 8.803697625993713e-07, 'epoch': 0.004610794684024953, 'num_input_tokens_seen': 2132803584, 'completed': '27.58% (1_017 / 3_687)', 'remaining time': '23:15:46', 'throughput': '7781.76', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:28:57,726 >> {'loss': 0.62, 'grad_norm': 15.50483226776123, 'learning_rate': 8.800955326084487e-07, 'epoch': 0.004882017900732303, 'num_input_tokens_seen': 2134900736, 'completed': '27.61% (1_018 / 3_687)', 'remaining time': '23:09:33', 'throughput': '9019.51', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:29:26,842 >> {'loss': 0.255, 'grad_norm': 9.97493839263916, 'learning_rate': 8.798210369692025e-07, 'epoch': 0.005153241117439653, 'num_input_tokens_seen': 2136997888, 'completed': '27.64% (1_019 / 3_687)', 'remaining time': '23:04:04', 'throughput': '9003.38', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:29:56,337 >> {'loss': 0.5253, 'grad_norm': 13.728446960449219, 'learning_rate': 8.795462759025364e-07, 'epoch': 0.005424464334147003, 'num_input_tokens_seen': 2139095040, 'completed': '27.66% (1_020 / 3_687)', 'remaining time': '22:59:56', 'throughput': '8887.74', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:30:28,143 >> {'loss': 0.3554, 'grad_norm': 13.042479515075684, 'learning_rate': 8.792712496295677e-07, 'epoch': 0.005695687550854353, 'num_input_tokens_seen': 2141192192, 'completed': '27.69% (1_021 / 3_687)', 'remaining time': '23:01:01', 'throughput': '8242.02', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:30:58,684 >> {'loss': 0.4398, 'grad_norm': 14.499848365783691, 'learning_rate': 8.789959583716268e-07, 'epoch': 0.005966910767561703, 'num_input_tokens_seen': 2143289344, 'completed': '27.72% (1_022 / 3_687)', 'remaining time': '22:59:25', 'throughput': '8583.25', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:31:28,795 >> {'loss': 0.3154, 'grad_norm': 12.071355819702148, 'learning_rate': 8.787204023502579e-07, 'epoch': 0.006238133984269053, 'num_input_tokens_seen': 2145386496, 'completed': '27.75% (1_023 / 3_687)', 'remaining time': '22:57:04', 'throughput': '8706.06', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:31:56,066 >> {'loss': 0.4772, 'grad_norm': 16.29543113708496, 'learning_rate': 8.78444581787218e-07, 'epoch': 0.006509357200976403, 'num_input_tokens_seen': 2147483648, 'completed': '27.77% (1_024 / 3_687)', 'remaining time': '22:49:38', 'throughput': '9612.56', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:32:28,517 >> {'loss': 0.5242, 'grad_norm': 14.60418701171875, 'learning_rate': 8.781684969044769e-07, 'epoch': 0.0067805804176837535, 'num_input_tokens_seen': 2149580800, 'completed': '27.80% (1_025 / 3_687)', 'remaining time': '22:51:56', 'throughput': '8078.20', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:33:00,452 >> {'loss': 0.6014, 'grad_norm': 16.060916900634766, 'learning_rate': 8.778921479242173e-07, 'epoch': 0.007051803634391104, 'num_input_tokens_seen': 2151677952, 'completed': '27.83% (1_026 / 3_687)', 'remaining time': '22:53:09', 'throughput': '8208.63', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:33:30,816 >> {'loss': 0.2425, 'grad_norm': 11.258378028869629, 'learning_rate': 8.776155350688342e-07, 'epoch': 0.007323026851098454, 'num_input_tokens_seen': 2153775104, 'completed': '27.85% (1_027 / 3_687)', 'remaining time': '22:51:39', 'throughput': '8633.45', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:34:00,846 >> {'loss': 0.3129, 'grad_norm': 13.625775337219238, 'learning_rate': 8.773386585609352e-07, 'epoch': 0.007594250067805804, 'num_input_tokens_seen': 2155872256, 'completed': '27.88% (1_028 / 3_687)', 'remaining time': '22:49:42', 'throughput': '8729.27', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:34:30,215 >> {'loss': 0.4769, 'grad_norm': 14.161125183105469, 'learning_rate': 8.770615186233398e-07, 'epoch': 0.007865473284513154, 'num_input_tokens_seen': 2157969408, 'completed': '27.91% (1_029 / 3_687)', 'remaining time': '22:46:50', 'throughput': '8926.03', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:35:00,692 >> {'loss': 0.4059, 'grad_norm': 14.752073287963867, 'learning_rate': 8.7678411547908e-07, 'epoch': 0.008136696501220505, 'num_input_tokens_seen': 2160066560, 'completed': '27.94% (1_030 / 3_687)', 'remaining time': '22:45:46', 'throughput': '8601.31', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:35:29,350 >> {'loss': 0.9232, 'grad_norm': 19.5262451171875, 'learning_rate': 8.76506449351399e-07, 'epoch': 0.008407919717927854, 'num_input_tokens_seen': 2162163712, 'completed': '27.96% (1_031 / 3_687)', 'remaining time': '22:42:08', 'throughput': '9147.26', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:36:01,590 >> {'loss': 0.3903, 'grad_norm': 23.627025604248047, 'learning_rate': 8.762285204637522e-07, 'epoch': 0.008679142934635205, 'num_input_tokens_seen': 2164260864, 'completed': '27.99% (1_032 / 3_687)', 'remaining time': '22:43:39', 'throughput': '8131.00', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:36:29,479 >> {'loss': 0.9677, 'grad_norm': 22.0424861907959, 'learning_rate': 8.75950329039806e-07, 'epoch': 0.008950366151342554, 'num_input_tokens_seen': 2166358016, 'completed': '28.02% (1_033 / 3_687)', 'remaining time': '22:39:12', 'throughput': '9399.68', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:37:00,579 >> {'loss': 0.4666, 'grad_norm': 16.031373977661133, 'learning_rate': 8.756718753034381e-07, 'epoch': 0.009221589368049905, 'num_input_tokens_seen': 2168455168, 'completed': '28.04% (1_034 / 3_687)', 'remaining time': '22:39:11', 'throughput': '8428.90', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:37:31,780 >> {'loss': 0.4239, 'grad_norm': 12.060325622558594, 'learning_rate': 8.75393159478738e-07, 'epoch': 0.009492812584757255, 'num_input_tokens_seen': 2170552320, 'completed': '28.07% (1_035 / 3_687)', 'remaining time': '22:39:15', 'throughput': '8401.79', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:38:02,110 >> {'loss': 0.5839, 'grad_norm': 17.0775089263916, 'learning_rate': 8.751141817900052e-07, 'epoch': 0.009764035801464606, 'num_input_tokens_seen': 2172649472, 'completed': '28.10% (1_036 / 3_687)', 'remaining time': '22:38:13', 'throughput': '8643.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:38:35,023 >> {'loss': 0.3156, 'grad_norm': 12.101827621459961, 'learning_rate': 8.748349424617504e-07, 'epoch': 0.010035259018171955, 'num_input_tokens_seen': 2174746624, 'completed': '28.13% (1_037 / 3_687)', 'remaining time': '22:40:18', 'throughput': '7964.89', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:39:06,013 >> {'loss': 0.7375, 'grad_norm': 19.348678588867188, 'learning_rate': 8.745554417186946e-07, 'epoch': 0.010306482234879306, 'num_input_tokens_seen': 2176843776, 'completed': '28.15% (1_038 / 3_687)', 'remaining time': '22:40:00', 'throughput': '8458.93', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:39:35,175 >> {'loss': 0.7549, 'grad_norm': 20.518571853637695, 'learning_rate': 8.742756797857698e-07, 'epoch': 0.010577705451586655, 'num_input_tokens_seen': 2178940928, 'completed': '28.18% (1_039 / 3_687)', 'remaining time': '22:37:38', 'throughput': '8989.30', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:40:07,171 >> {'loss': 0.5175, 'grad_norm': 29.709476470947266, 'learning_rate': 8.739956568881174e-07, 'epoch': 0.010848928668294006, 'num_input_tokens_seen': 2181038080, 'completed': '28.21% (1_040 / 3_687)', 'remaining time': '22:38:29', 'throughput': '8193.02', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:40:39,040 >> {'loss': 0.5272, 'grad_norm': 16.73153305053711, 'learning_rate': 8.737153732510894e-07, 'epoch': 0.011120151885001357, 'num_input_tokens_seen': 2183135232, 'completed': '28.23% (1_041 / 3_687)', 'remaining time': '22:39:08', 'throughput': '8225.66', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:41:08,249 >> {'loss': 0.4049, 'grad_norm': 11.584559440612793, 'learning_rate': 8.734348291002472e-07, 'epoch': 0.011391375101708706, 'num_input_tokens_seen': 2185232384, 'completed': '28.26% (1_042 / 3_687)', 'remaining time': '22:36:55', 'throughput': '8974.70', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:41:39,763 >> {'loss': 0.6874, 'grad_norm': 16.299226760864258, 'learning_rate': 8.731540246613621e-07, 'epoch': 0.011662598318416057, 'num_input_tokens_seen': 2187329536, 'completed': '28.29% (1_043 / 3_687)', 'remaining time': '22:37:10', 'throughput': '8318.21', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:42:10,729 >> {'loss': 0.2929, 'grad_norm': 12.125165939331055, 'learning_rate': 8.728729601604149e-07, 'epoch': 0.011933821535123406, 'num_input_tokens_seen': 2189426688, 'completed': '28.32% (1_044 / 3_687)', 'remaining time': '22:36:49', 'throughput': '8466.06', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:42:41,551 >> {'loss': 0.5507, 'grad_norm': 15.17818546295166, 'learning_rate': 8.725916358235956e-07, 'epoch': 0.012205044751830757, 'num_input_tokens_seen': 2191523840, 'completed': '28.34% (1_045 / 3_687)', 'remaining time': '22:36:19', 'throughput': '8504.81', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:43:08,687 >> {'loss': 0.7183, 'grad_norm': 15.40369987487793, 'learning_rate': 8.723100518773034e-07, 'epoch': 0.012476267968538107, 'num_input_tokens_seen': 2193620992, 'completed': '28.37% (1_046 / 3_687)', 'remaining time': '22:32:18', 'throughput': '9660.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:43:39,016 >> {'loss': 0.3474, 'grad_norm': 13.198980331420898, 'learning_rate': 8.720282085481463e-07, 'epoch': 0.012747491185245458, 'num_input_tokens_seen': 2195718144, 'completed': '28.40% (1_047 / 3_687)', 'remaining time': '22:31:25', 'throughput': '8643.50', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:44:10,472 >> {'loss': 0.4623, 'grad_norm': 13.224248886108398, 'learning_rate': 8.717461060629408e-07, 'epoch': 0.013018714401952807, 'num_input_tokens_seen': 2197815296, 'completed': '28.42% (1_048 / 3_687)', 'remaining time': '22:31:35', 'throughput': '8333.54', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:44:41,473 >> {'loss': 0.6044, 'grad_norm': 21.415082931518555, 'learning_rate': 8.714637446487127e-07, 'epoch': 0.013289937618660158, 'num_input_tokens_seen': 2199912448, 'completed': '28.45% (1_049 / 3_687)', 'remaining time': '22:31:19', 'throughput': '8456.40', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:45:11,575 >> {'loss': 0.4799, 'grad_norm': 17.107685089111328, 'learning_rate': 8.711811245326955e-07, 'epoch': 0.013561160835367507, 'num_input_tokens_seen': 2202009600, 'completed': '28.48% (1_050 / 3_687)', 'remaining time': '22:30:15', 'throughput': '8708.24', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:45:44,712 >> {'loss': 0.5492, 'grad_norm': 16.153474807739258, 'learning_rate': 8.70898245942331e-07, 'epoch': 0.013832384052074858, 'num_input_tokens_seen': 2204106752, 'completed': '28.51% (1_051 / 3_687)', 'remaining time': '22:31:49', 'throughput': '7910.73', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:46:13,943 >> {'loss': 0.6168, 'grad_norm': 14.7495756149292, 'learning_rate': 8.706151091052693e-07, 'epoch': 0.014103607268782207, 'num_input_tokens_seen': 2206203904, 'completed': '28.53% (1_052 / 3_687)', 'remaining time': '22:30:00', 'throughput': '8968.00', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:46:46,168 >> {'loss': 0.5481, 'grad_norm': 15.120250701904297, 'learning_rate': 8.703317142493681e-07, 'epoch': 0.014374830485489558, 'num_input_tokens_seen': 2208301056, 'completed': '28.56% (1_053 / 3_687)', 'remaining time': '22:30:43', 'throughput': '8134.84', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:47:14,235 >> {'loss': 0.6981, 'grad_norm': 18.195310592651367, 'learning_rate': 8.700480616026928e-07, 'epoch': 0.014646053702196907, 'num_input_tokens_seen': 2210398208, 'completed': '28.59% (1_054 / 3_687)', 'remaining time': '22:28:01', 'throughput': '9340.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:47:45,152 >> {'loss': 0.5543, 'grad_norm': 15.834187507629395, 'learning_rate': 8.697641513935164e-07, 'epoch': 0.014917276918904258, 'num_input_tokens_seen': 2212495360, 'completed': '28.61% (1_055 / 3_687)', 'remaining time': '22:27:40', 'throughput': '8478.84', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:48:14,541 >> {'loss': 0.5255, 'grad_norm': 13.692312240600586, 'learning_rate': 8.694799838503186e-07, 'epoch': 0.015188500135611608, 'num_input_tokens_seen': 2214592512, 'completed': '28.64% (1_056 / 3_687)', 'remaining time': '22:26:06', 'throughput': '8919.83', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:48:44,733 >> {'loss': 0.3443, 'grad_norm': 11.233868598937988, 'learning_rate': 8.691955592017872e-07, 'epoch': 0.015459723352318959, 'num_input_tokens_seen': 2216689664, 'completed': '28.67% (1_057 / 3_687)', 'remaining time': '22:25:12', 'throughput': '8682.44', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:49:10,235 >> {'loss': 0.7494, 'grad_norm': 20.017642974853516, 'learning_rate': 8.689108776768159e-07, 'epoch': 0.015730946569026308, 'num_input_tokens_seen': 2218786816, 'completed': '28.70% (1_058 / 3_687)', 'remaining time': '22:20:47', 'throughput': '10279.44', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:49:39,376 >> {'loss': 0.6063, 'grad_norm': 14.512331008911133, 'learning_rate': 8.686259395045056e-07, 'epoch': 0.01600216978573366, 'num_input_tokens_seen': 2220883968, 'completed': '28.72% (1_059 / 3_687)', 'remaining time': '22:19:11', 'throughput': '8995.75', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:50:11,041 >> {'loss': 0.4511, 'grad_norm': 15.613311767578125, 'learning_rate': 8.68340744914164e-07, 'epoch': 0.01627339300244101, 'num_input_tokens_seen': 2222981120, 'completed': '28.75% (1_060 / 3_687)', 'remaining time': '22:19:28', 'throughput': '8278.60', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:50:41,348 >> {'loss': 0.3019, 'grad_norm': 13.627348899841309, 'learning_rate': 8.680552941353045e-07, 'epoch': 0.01654461621914836, 'num_input_tokens_seen': 2225078272, 'completed': '28.78% (1_061 / 3_687)', 'remaining time': '22:18:45', 'throughput': '8649.63', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:51:11,986 >> {'loss': 0.701, 'grad_norm': 16.624229431152344, 'learning_rate': 8.677695873976473e-07, 'epoch': 0.016815839435855708, 'num_input_tokens_seen': 2227175424, 'completed': '28.80% (1_062 / 3_687)', 'remaining time': '22:18:17', 'throughput': '8556.23', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:51:42,246 >> {'loss': 0.4085, 'grad_norm': 11.855605125427246, 'learning_rate': 8.674836249311182e-07, 'epoch': 0.01708706265256306, 'num_input_tokens_seen': 2229272576, 'completed': '28.83% (1_063 / 3_687)', 'remaining time': '22:17:32', 'throughput': '8663.36', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:52:09,333 >> {'loss': 0.6829, 'grad_norm': 16.31574249267578, 'learning_rate': 8.671974069658488e-07, 'epoch': 0.01735828586927041, 'num_input_tokens_seen': 2231369728, 'completed': '28.86% (1_064 / 3_687)', 'remaining time': '22:14:38', 'throughput': '9677.36', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:52:37,866 >> {'loss': 0.603, 'grad_norm': 16.633039474487305, 'learning_rate': 8.669109337321767e-07, 'epoch': 0.01762950908597776, 'num_input_tokens_seen': 2233466880, 'completed': '28.89% (1_065 / 3_687)', 'remaining time': '22:12:47', 'throughput': '9187.40', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:53:06,622 >> {'loss': 0.4269, 'grad_norm': 12.263994216918945, 'learning_rate': 8.666242054606444e-07, 'epoch': 0.01790073230268511, 'num_input_tokens_seen': 2235564032, 'completed': '28.91% (1_066 / 3_687)', 'remaining time': '22:11:08', 'throughput': '9116.34', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:53:39,113 >> {'loss': 0.5071, 'grad_norm': 13.086897850036621, 'learning_rate': 8.66337222382e-07, 'epoch': 0.01817195551939246, 'num_input_tokens_seen': 2237661184, 'completed': '28.94% (1_067 / 3_687)', 'remaining time': '22:11:56', 'throughput': '8068.37', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:54:08,289 >> {'loss': 0.7171, 'grad_norm': 18.24944305419922, 'learning_rate': 8.660499847271965e-07, 'epoch': 0.01844317873609981, 'num_input_tokens_seen': 2239758336, 'completed': '28.97% (1_068 / 3_687)', 'remaining time': '22:10:35', 'throughput': '8984.57', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:54:39,778 >> {'loss': 0.3733, 'grad_norm': 12.880998611450195, 'learning_rate': 8.657624927273919e-07, 'epoch': 0.01871440195280716, 'num_input_tokens_seen': 2241855488, 'completed': '28.99% (1_069 / 3_687)', 'remaining time': '22:10:42', 'throughput': '8325.07', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:55:09,157 >> {'loss': 0.417, 'grad_norm': 11.316634178161621, 'learning_rate': 8.654747466139488e-07, 'epoch': 0.01898562516951451, 'num_input_tokens_seen': 2243952640, 'completed': '29.02% (1_070 / 3_687)', 'remaining time': '22:09:30', 'throughput': '8922.85', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:55:37,977 >> {'loss': 0.515, 'grad_norm': 16.127735137939453, 'learning_rate': 8.651867466184344e-07, 'epoch': 0.01925684838622186, 'num_input_tokens_seen': 2246049792, 'completed': '29.05% (1_071 / 3_687)', 'remaining time': '22:07:58', 'throughput': '9095.89', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:56:08,540 >> {'loss': 0.4598, 'grad_norm': 13.648701667785645, 'learning_rate': 8.6489849297262e-07, 'epoch': 0.01952807160292921, 'num_input_tokens_seen': 2248146944, 'completed': '29.08% (1_072 / 3_687)', 'remaining time': '22:07:32', 'throughput': '8577.02', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:56:38,440 >> {'loss': 0.4691, 'grad_norm': 18.77680778503418, 'learning_rate': 8.646099859084812e-07, 'epoch': 0.019799294819636562, 'num_input_tokens_seen': 2250244096, 'completed': '29.10% (1_073 / 3_687)', 'remaining time': '22:06:41', 'throughput': '8767.53', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:57:07,219 >> {'loss': 0.4609, 'grad_norm': 14.230118751525879, 'learning_rate': 8.643212256581978e-07, 'epoch': 0.02007051803634391, 'num_input_tokens_seen': 2252341248, 'completed': '29.13% (1_074 / 3_687)', 'remaining time': '22:05:12', 'throughput': '9108.71', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:57:36,902 >> {'loss': 0.4302, 'grad_norm': 20.32296371459961, 'learning_rate': 8.640322124541525e-07, 'epoch': 0.02034174125305126, 'num_input_tokens_seen': 2254438400, 'completed': '29.16% (1_075 / 3_687)', 'remaining time': '22:04:15', 'throughput': '8831.58', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:58:06,360 >> {'loss': 0.7439, 'grad_norm': 19.631397247314453, 'learning_rate': 8.637429465289324e-07, 'epoch': 0.02061296446975861, 'num_input_tokens_seen': 2256535552, 'completed': '29.18% (1_076 / 3_687)', 'remaining time': '22:03:12', 'throughput': '8898.82', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:58:34,622 >> {'loss': 0.9491, 'grad_norm': 20.95248794555664, 'learning_rate': 8.63453428115328e-07, 'epoch': 0.020884187686465962, 'num_input_tokens_seen': 2258632704, 'completed': '29.21% (1_077 / 3_687)', 'remaining time': '22:01:29', 'throughput': '9275.54', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:59:01,850 >> {'loss': 1.0095, 'grad_norm': 23.594327926635742, 'learning_rate': 8.631636574463321e-07, 'epoch': 0.02115541090317331, 'num_input_tokens_seen': 2260729856, 'completed': '29.24% (1_078 / 3_687)', 'remaining time': '21:59:13', 'throughput': '9627.86', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 02:59:31,093 >> {'loss': 0.4976, 'grad_norm': 13.66223430633545, 'learning_rate': 8.628736347551417e-07, 'epoch': 0.02142663411988066, 'num_input_tokens_seen': 2262827008, 'completed': '29.26% (1_079 / 3_687)', 'remaining time': '21:58:06', 'throughput': '8964.25', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:00:04,207 >> {'loss': 0.4426, 'grad_norm': 14.790290832519531, 'learning_rate': 8.625833602751559e-07, 'epoch': 0.021697857336588012, 'num_input_tokens_seen': 2264924160, 'completed': '29.29% (1_080 / 3_687)', 'remaining time': '21:59:07', 'throughput': '7916.41', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:00:33,068 >> {'loss': 0.9969, 'grad_norm': 22.70588493347168, 'learning_rate': 8.622928342399762e-07, 'epoch': 0.021969080553295363, 'num_input_tokens_seen': 2267021312, 'completed': '29.32% (1_081 / 3_687)', 'remaining time': '21:57:48', 'throughput': '9082.98', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:01:02,657 >> {'loss': 0.6167, 'grad_norm': 15.693807601928711, 'learning_rate': 8.620020568834072e-07, 'epoch': 0.022240303770002714, 'num_input_tokens_seen': 2269118464, 'completed': '29.35% (1_082 / 3_687)', 'remaining time': '21:56:54', 'throughput': '8859.53', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:01:31,264 >> {'loss': 0.5203, 'grad_norm': 14.573871612548828, 'learning_rate': 8.617110284394553e-07, 'epoch': 0.02251152698671006, 'num_input_tokens_seen': 2271215616, 'completed': '29.37% (1_083 / 3_687)', 'remaining time': '21:55:30', 'throughput': '9163.43', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:02:00,401 >> {'loss': 0.3329, 'grad_norm': 12.033546447753906, 'learning_rate': 8.614197491423293e-07, 'epoch': 0.022782750203417412, 'num_input_tokens_seen': 2273312768, 'completed': '29.40% (1_084 / 3_687)', 'remaining time': '21:54:23', 'throughput': '8997.07', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:02:29,023 >> {'loss': 0.8502, 'grad_norm': 21.395675659179688, 'learning_rate': 8.611282192264396e-07, 'epoch': 0.023053973420124763, 'num_input_tokens_seen': 2275409920, 'completed': '29.43% (1_085 / 3_687)', 'remaining time': '21:53:01', 'throughput': '9158.90', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:02:58,945 >> {'loss': 0.3252, 'grad_norm': 10.336832046508789, 'learning_rate': 8.608364389263984e-07, 'epoch': 0.023325196636832114, 'num_input_tokens_seen': 2277507072, 'completed': '29.45% (1_086 / 3_687)', 'remaining time': '21:52:20', 'throughput': '8760.86', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:03:31,179 >> {'loss': 0.4895, 'grad_norm': 14.438992500305176, 'learning_rate': 8.605444084770192e-07, 'epoch': 0.023596419853539462, 'num_input_tokens_seen': 2279604224, 'completed': '29.48% (1_087 / 3_687)', 'remaining time': '21:52:49', 'throughput': '8132.54', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:04:01,815 >> {'loss': 0.4126, 'grad_norm': 12.839303970336914, 'learning_rate': 8.602521281133173e-07, 'epoch': 0.023867643070246813, 'num_input_tokens_seen': 2281701376, 'completed': '29.51% (1_088 / 3_687)', 'remaining time': '21:52:28', 'throughput': '8556.86', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:04:32,077 >> {'loss': 0.298, 'grad_norm': 15.832900047302246, 'learning_rate': 8.599595980705085e-07, 'epoch': 0.024138866286954164, 'num_input_tokens_seen': 2283798528, 'completed': '29.54% (1_089 / 3_687)', 'remaining time': '21:51:57', 'throughput': '8662.42', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:05:04,652 >> {'loss': 0.5055, 'grad_norm': 16.126102447509766, 'learning_rate': 8.596668185840102e-07, 'epoch': 0.024410089503661515, 'num_input_tokens_seen': 2285895680, 'completed': '29.56% (1_090 / 3_687)', 'remaining time': '21:52:32', 'throughput': '8047.42', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:05:32,673 >> {'loss': 0.5144, 'grad_norm': 13.678728103637695, 'learning_rate': 8.593737898894398e-07, 'epoch': 0.024681312720368862, 'num_input_tokens_seen': 2287992832, 'completed': '29.59% (1_091 / 3_687)', 'remaining time': '21:50:56', 'throughput': '9355.21', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:05:59,558 >> {'loss': 0.5175, 'grad_norm': 13.563033103942871, 'learning_rate': 8.59080512222616e-07, 'epoch': 0.024952535937076213, 'num_input_tokens_seen': 2290089984, 'completed': '29.62% (1_092 / 3_687)', 'remaining time': '21:48:50', 'throughput': '9750.51', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:06:33,080 >> {'loss': 0.662, 'grad_norm': 22.787578582763672, 'learning_rate': 8.587869858195574e-07, 'epoch': 0.025223759153783564, 'num_input_tokens_seen': 2292187136, 'completed': '29.64% (1_093 / 3_687)', 'remaining time': '21:49:50', 'throughput': '7820.14', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:07:01,780 >> {'loss': 0.4915, 'grad_norm': 12.561062812805176, 'learning_rate': 8.584932109164826e-07, 'epoch': 0.025494982370490915, 'num_input_tokens_seen': 2294284288, 'completed': '29.67% (1_094 / 3_687)', 'remaining time': '21:48:36', 'throughput': '9133.75', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:07:35,254 >> {'loss': 0.3419, 'grad_norm': 10.324007034301758, 'learning_rate': 8.581991877498109e-07, 'epoch': 0.025766205587198263, 'num_input_tokens_seen': 2296381440, 'completed': '29.70% (1_095 / 3_687)', 'remaining time': '21:49:33', 'throughput': '7831.36', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:08:03,704 >> {'loss': 0.5506, 'grad_norm': 16.793432235717773, 'learning_rate': 8.579049165561607e-07, 'epoch': 0.026037428803905614, 'num_input_tokens_seen': 2298478592, 'completed': '29.73% (1_096 / 3_687)', 'remaining time': '21:48:12', 'throughput': '9214.26', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:08:31,947 >> {'loss': 0.6403, 'grad_norm': 18.08753776550293, 'learning_rate': 8.576103975723502e-07, 'epoch': 0.026308652020612965, 'num_input_tokens_seen': 2300575744, 'completed': '29.75% (1_097 / 3_687)', 'remaining time': '21:46:47', 'throughput': '9281.67', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:09:00,488 >> {'loss': 0.5685, 'grad_norm': 14.949502944946289, 'learning_rate': 8.573156310353974e-07, 'epoch': 0.026579875237320316, 'num_input_tokens_seen': 2302672896, 'completed': '29.78% (1_098 / 3_687)', 'remaining time': '21:45:31', 'throughput': '9184.78', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:09:27,358 >> {'loss': 0.5644, 'grad_norm': 14.36096477508545, 'learning_rate': 8.570206171825188e-07, 'epoch': 0.026851098454027666, 'num_input_tokens_seen': 2304770048, 'completed': '29.81% (1_099 / 3_687)', 'remaining time': '21:43:32', 'throughput': '9755.85', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:09:57,340 >> {'loss': 0.439, 'grad_norm': 13.551562309265137, 'learning_rate': 8.567253562511306e-07, 'epoch': 0.027122321670735014, 'num_input_tokens_seen': 2306867200, 'completed': '29.83% (1_100 / 3_687)', 'remaining time': '21:42:56', 'throughput': '8743.39', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:10:26,308 >> {'loss': 0.4937, 'grad_norm': 18.999597549438477, 'learning_rate': 8.564298484788472e-07, 'epoch': 0.027393544887442365, 'num_input_tokens_seen': 2308964352, 'completed': '29.86% (1_101 / 3_687)', 'remaining time': '21:41:54', 'throughput': '9049.65', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:10:56,945 >> {'loss': 0.2738, 'grad_norm': 10.107467651367188, 'learning_rate': 8.561340941034825e-07, 'epoch': 0.027664768104149716, 'num_input_tokens_seen': 2311061504, 'completed': '29.89% (1_102 / 3_687)', 'remaining time': '21:41:34', 'throughput': '8556.30', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:11:27,991 >> {'loss': 0.6454, 'grad_norm': 19.000905990600586, 'learning_rate': 8.55838093363048e-07, 'epoch': 0.027935991320857067, 'num_input_tokens_seen': 2313158656, 'completed': '29.92% (1_103 / 3_687)', 'remaining time': '21:41:25', 'throughput': '8443.69', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:11:59,402 >> {'loss': 0.6641, 'grad_norm': 31.94487190246582, 'learning_rate': 8.555418464957542e-07, 'epoch': 0.028207214537564414, 'num_input_tokens_seen': 2315255808, 'completed': '29.94% (1_104 / 3_687)', 'remaining time': '21:41:25', 'throughput': '8345.60', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:12:28,424 >> {'loss': 0.4975, 'grad_norm': 16.993053436279297, 'learning_rate': 8.552453537400089e-07, 'epoch': 0.028478437754271765, 'num_input_tokens_seen': 2317352960, 'completed': '29.97% (1_105 / 3_687)', 'remaining time': '21:40:25', 'throughput': '9032.67', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:13:00,207 >> {'loss': 0.7325, 'grad_norm': 21.718708038330078, 'learning_rate': 8.549486153344183e-07, 'epoch': 0.028749660970979116, 'num_input_tokens_seen': 2319450112, 'completed': '30.00% (1_106 / 3_687)', 'remaining time': '21:40:32', 'throughput': '8247.94', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:13:27,379 >> {'loss': 0.5915, 'grad_norm': 16.96457862854004, 'learning_rate': 8.546516315177863e-07, 'epoch': 0.029020884187686467, 'num_input_tokens_seen': 2321547264, 'completed': '30.02% (1_107 / 3_687)', 'remaining time': '21:38:48', 'throughput': '9647.76', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:13:58,299 >> {'loss': 0.5334, 'grad_norm': 15.942675590515137, 'learning_rate': 8.543544025291143e-07, 'epoch': 0.029292107404393815, 'num_input_tokens_seen': 2323644416, 'completed': '30.05% (1_108 / 3_687)', 'remaining time': '21:38:35', 'throughput': '8478.04', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:14:28,559 >> {'loss': 0.5069, 'grad_norm': 14.432857513427734, 'learning_rate': 8.540569286076004e-07, 'epoch': 0.029563330621101166, 'num_input_tokens_seen': 2325741568, 'completed': '30.08% (1_109 / 3_687)', 'remaining time': '21:38:06', 'throughput': '8663.14', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:14:59,622 >> {'loss': 0.6163, 'grad_norm': 16.693790435791016, 'learning_rate': 8.537592099926407e-07, 'epoch': 0.029834553837808517, 'num_input_tokens_seen': 2327838720, 'completed': '30.11% (1_110 / 3_687)', 'remaining time': '21:37:56', 'throughput': '8439.12', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:15:31,233 >> {'loss': 0.3806, 'grad_norm': 13.320590019226074, 'learning_rate': 8.534612469238278e-07, 'epoch': 0.030105777054515868, 'num_input_tokens_seen': 2329935872, 'completed': '30.13% (1_111 / 3_687)', 'remaining time': '21:37:58', 'throughput': '8292.62', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:16:01,388 >> {'loss': 0.6614, 'grad_norm': 20.88117790222168, 'learning_rate': 8.531630396409507e-07, 'epoch': 0.030377000271223215, 'num_input_tokens_seen': 2332033024, 'completed': '30.16% (1_112 / 3_687)', 'remaining time': '21:37:26', 'throughput': '8693.44', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:16:32,314 >> {'loss': 0.489, 'grad_norm': 12.577048301696777, 'learning_rate': 8.528645883839956e-07, 'epoch': 0.030648223487930566, 'num_input_tokens_seen': 2334130176, 'completed': '30.19% (1_113 / 3_687)', 'remaining time': '21:37:12', 'throughput': '8476.44', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:17:02,970 >> {'loss': 0.5687, 'grad_norm': 16.97111701965332, 'learning_rate': 8.525658933931448e-07, 'epoch': 0.030919446704637917, 'num_input_tokens_seen': 2336227328, 'completed': '30.21% (1_114 / 3_687)', 'remaining time': '21:36:51', 'throughput': '8551.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:17:29,609 >> {'loss': 0.556, 'grad_norm': 15.8145112991333, 'learning_rate': 8.522669549087762e-07, 'epoch': 0.031190669921345268, 'num_input_tokens_seen': 2338324480, 'completed': '30.24% (1_115 / 3_687)', 'remaining time': '21:35:00', 'throughput': '9840.45', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:17:59,834 >> {'loss': 0.7402, 'grad_norm': 16.058319091796875, 'learning_rate': 8.519677731714645e-07, 'epoch': 0.031461893138052616, 'num_input_tokens_seen': 2340421632, 'completed': '30.27% (1_116 / 3_687)', 'remaining time': '21:34:30', 'throughput': '8673.12', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:18:29,249 >> {'loss': 0.4213, 'grad_norm': 11.520198822021484, 'learning_rate': 8.516683484219797e-07, 'epoch': 0.03173311635475997, 'num_input_tokens_seen': 2342518784, 'completed': '30.30% (1_117 / 3_687)', 'remaining time': '21:33:42', 'throughput': '8911.81', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:18:59,584 >> {'loss': 0.5229, 'grad_norm': 18.839998245239258, 'learning_rate': 8.513686809012875e-07, 'epoch': 0.03200433957146732, 'num_input_tokens_seen': 2344615936, 'completed': '30.32% (1_118 / 3_687)', 'remaining time': '21:33:15', 'throughput': '8641.83', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:19:28,717 >> {'loss': 0.5792, 'grad_norm': 16.22575569152832, 'learning_rate': 8.510687708505489e-07, 'epoch': 0.032275562788174665, 'num_input_tokens_seen': 2346713088, 'completed': '30.35% (1_119 / 3_687)', 'remaining time': '21:32:22', 'throughput': '8998.20', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:19:59,019 >> {'loss': 0.5386, 'grad_norm': 15.674661636352539, 'learning_rate': 8.507686185111199e-07, 'epoch': 0.03254678600488202, 'num_input_tokens_seen': 2348810240, 'completed': '30.38% (1_120 / 3_687)', 'remaining time': '21:31:54', 'throughput': '8651.08', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:20:28,089 >> {'loss': 0.3716, 'grad_norm': 18.9416446685791, 'learning_rate': 8.504682241245516e-07, 'epoch': 0.03281800922158937, 'num_input_tokens_seen': 2350907392, 'completed': '30.40% (1_121 / 3_687)', 'remaining time': '21:31:00', 'throughput': '9017.52', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:20:58,384 >> {'loss': 0.2198, 'grad_norm': 10.184718132019043, 'learning_rate': 8.501675879325906e-07, 'epoch': 0.03308923243829672, 'num_input_tokens_seen': 2353004544, 'completed': '30.43% (1_122 / 3_687)', 'remaining time': '21:30:32', 'throughput': '8653.08', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:21:29,450 >> {'loss': 0.4381, 'grad_norm': 12.775616645812988, 'learning_rate': 8.498667101771769e-07, 'epoch': 0.03336045565500407, 'num_input_tokens_seen': 2355101696, 'completed': '30.46% (1_123 / 3_687)', 'remaining time': '21:30:20', 'throughput': '8438.43', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:22:02,910 >> {'loss': 0.6905, 'grad_norm': 19.200225830078125, 'learning_rate': 8.495655911004456e-07, 'epoch': 0.033631678871711417, 'num_input_tokens_seen': 2357198848, 'completed': '30.49% (1_124 / 3_687)', 'remaining time': '21:30:57', 'throughput': '7834.53', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:22:33,511 >> {'loss': 0.551, 'grad_norm': 16.312833786010742, 'learning_rate': 8.492642309447257e-07, 'epoch': 0.03390290208841877, 'num_input_tokens_seen': 2359296000, 'completed': '30.51% (1_125 / 3_687)', 'remaining time': '21:30:35', 'throughput': '8566.45', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:23:03,300 >> {'loss': 0.3892, 'grad_norm': 12.200955390930176, 'learning_rate': 8.489626299525409e-07, 'epoch': 0.03417412530512612, 'num_input_tokens_seen': 2361393152, 'completed': '30.54% (1_126 / 3_687)', 'remaining time': '21:29:56', 'throughput': '8799.94', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:23:32,934 >> {'loss': 0.4058, 'grad_norm': 12.41970157623291, 'learning_rate': 8.486607883666077e-07, 'epoch': 0.034445348521833466, 'num_input_tokens_seen': 2363490304, 'completed': '30.57% (1_127 / 3_687)', 'remaining time': '21:29:14', 'throughput': '8845.95', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:24:03,941 >> {'loss': 0.5063, 'grad_norm': 17.284793853759766, 'learning_rate': 8.483587064298372e-07, 'epoch': 0.03471657173854082, 'num_input_tokens_seen': 2365587456, 'completed': '30.59% (1_128 / 3_687)', 'remaining time': '21:28:59', 'throughput': '8454.38', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:24:36,817 >> {'loss': 0.3901, 'grad_norm': 12.992026329040527, 'learning_rate': 8.480563843853328e-07, 'epoch': 0.03498779495524817, 'num_input_tokens_seen': 2367684608, 'completed': '30.62% (1_129 / 3_687)', 'remaining time': '21:29:22', 'throughput': '7973.69', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:25:07,653 >> {'loss': 0.6989, 'grad_norm': 25.536312103271484, 'learning_rate': 8.477538224763923e-07, 'epoch': 0.03525901817195552, 'num_input_tokens_seen': 2369781760, 'completed': '30.65% (1_130 / 3_687)', 'remaining time': '21:29:03', 'throughput': '8501.30', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:25:38,462 >> {'loss': 0.4448, 'grad_norm': 14.612037658691406, 'learning_rate': 8.474510209465058e-07, 'epoch': 0.03553024138866287, 'num_input_tokens_seen': 2371878912, 'completed': '30.68% (1_131 / 3_687)', 'remaining time': '21:28:44', 'throughput': '8508.80', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:26:09,146 >> {'loss': 0.5952, 'grad_norm': 17.197641372680664, 'learning_rate': 8.471479800393565e-07, 'epoch': 0.03580146460537022, 'num_input_tokens_seen': 2373976064, 'completed': '30.70% (1_132 / 3_687)', 'remaining time': '21:28:22', 'throughput': '8543.34', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:26:39,555 >> {'loss': 0.5679, 'grad_norm': 15.149774551391602, 'learning_rate': 8.468446999988202e-07, 'epoch': 0.03607268782207757, 'num_input_tokens_seen': 2376073216, 'completed': '30.73% (1_133 / 3_687)', 'remaining time': '21:27:55', 'throughput': '8620.49', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:27:05,122 >> {'loss': 0.9363, 'grad_norm': 20.161657333374023, 'learning_rate': 8.465411810689653e-07, 'epoch': 0.03634391103878492, 'num_input_tokens_seen': 2378170368, 'completed': '30.76% (1_134 / 3_687)', 'remaining time': '21:25:55', 'throughput': '10253.43', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:27:35,906 >> {'loss': 0.4595, 'grad_norm': 13.918910026550293, 'learning_rate': 8.462374234940517e-07, 'epoch': 0.03661513425549227, 'num_input_tokens_seen': 2380267520, 'completed': '30.78% (1_135 / 3_687)', 'remaining time': '21:25:35', 'throughput': '8515.53', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:28:05,325 >> {'loss': 0.3769, 'grad_norm': 11.941801071166992, 'learning_rate': 8.459334275185325e-07, 'epoch': 0.03688635747219962, 'num_input_tokens_seen': 2382364672, 'completed': '30.81% (1_136 / 3_687)', 'remaining time': '21:24:50', 'throughput': '8910.76', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:28:33,468 >> {'loss': 1.0131, 'grad_norm': 23.785158157348633, 'learning_rate': 8.456291933870521e-07, 'epoch': 0.03715758068890697, 'num_input_tokens_seen': 2384461824, 'completed': '30.84% (1_137 / 3_687)', 'remaining time': '21:23:41', 'throughput': '9314.52', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:29:04,895 >> {'loss': 0.2438, 'grad_norm': 12.919363021850586, 'learning_rate': 8.453247213444463e-07, 'epoch': 0.03742880390561432, 'num_input_tokens_seen': 2386558976, 'completed': '30.87% (1_138 / 3_687)', 'remaining time': '21:23:33', 'throughput': '8341.47', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:29:35,415 >> {'loss': 0.4749, 'grad_norm': 14.640851020812988, 'learning_rate': 8.450200116357428e-07, 'epoch': 0.03770002712232167, 'num_input_tokens_seen': 2388656128, 'completed': '30.89% (1_139 / 3_687)', 'remaining time': '21:23:09', 'throughput': '8589.20', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:30:07,406 >> {'loss': 0.3616, 'grad_norm': 10.543968200683594, 'learning_rate': 8.4471506450616e-07, 'epoch': 0.03797125033902902, 'num_input_tokens_seen': 2390753280, 'completed': '30.92% (1_140 / 3_687)', 'remaining time': '21:23:11', 'throughput': '8194.45', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:30:36,514 >> {'loss': 0.4972, 'grad_norm': 15.393962860107422, 'learning_rate': 8.444098802011083e-07, 'epoch': 0.03824247355573637, 'num_input_tokens_seen': 2392850432, 'completed': '30.95% (1_141 / 3_687)', 'remaining time': '21:22:21', 'throughput': '9005.76', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:31:06,857 >> {'loss': 0.5945, 'grad_norm': 79.32931518554688, 'learning_rate': 8.441044589661881e-07, 'epoch': 0.03851369677244372, 'num_input_tokens_seen': 2394947584, 'completed': '30.97% (1_142 / 3_687)', 'remaining time': '21:21:52', 'throughput': '8639.46', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:31:36,973 >> {'loss': 0.5593, 'grad_norm': 13.988775253295898, 'learning_rate': 8.437988010471907e-07, 'epoch': 0.038784919989151075, 'num_input_tokens_seen': 2397044736, 'completed': '31.00% (1_143 / 3_687)', 'remaining time': '21:21:20', 'throughput': '8704.44', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:32:06,915 >> {'loss': 0.5196, 'grad_norm': 14.631237983703613, 'learning_rate': 8.434929066900982e-07, 'epoch': 0.03905614320585842, 'num_input_tokens_seen': 2399141888, 'completed': '31.03% (1_144 / 3_687)', 'remaining time': '21:20:45', 'throughput': '8755.03', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:32:34,404 >> {'loss': 0.8291, 'grad_norm': 23.943798065185547, 'learning_rate': 8.431867761410826e-07, 'epoch': 0.03932736642256577, 'num_input_tokens_seen': 2401239040, 'completed': '31.06% (1_145 / 3_687)', 'remaining time': '21:19:27', 'throughput': '9536.41', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:33:03,670 >> {'loss': 0.4124, 'grad_norm': 13.034646034240723, 'learning_rate': 8.42880409646506e-07, 'epoch': 0.039598589639273124, 'num_input_tokens_seen': 2403336192, 'completed': '31.08% (1_146 / 3_687)', 'remaining time': '21:18:41', 'throughput': '8957.03', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:33:33,571 >> {'loss': 0.4623, 'grad_norm': 16.411720275878906, 'learning_rate': 8.42573807452921e-07, 'epoch': 0.03986981285598047, 'num_input_tokens_seen': 2405433344, 'completed': '31.11% (1_147 / 3_687)', 'remaining time': '21:18:05', 'throughput': '8767.35', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:34:02,372 >> {'loss': 0.5727, 'grad_norm': 18.51169204711914, 'learning_rate': 8.422669698070687e-07, 'epoch': 0.04014103607268782, 'num_input_tokens_seen': 2407530496, 'completed': '31.14% (1_148 / 3_687)', 'remaining time': '21:17:11', 'throughput': '9101.85', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:34:31,786 >> {'loss': 0.2515, 'grad_norm': 10.572891235351562, 'learning_rate': 8.419598969558808e-07, 'epoch': 0.040412259289395173, 'num_input_tokens_seen': 2409627648, 'completed': '31.16% (1_149 / 3_687)', 'remaining time': '21:16:28', 'throughput': '8912.08', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:35:02,943 >> {'loss': 0.3125, 'grad_norm': 10.220544815063477, 'learning_rate': 8.416525891464776e-07, 'epoch': 0.04068348250610252, 'num_input_tokens_seen': 2411724800, 'completed': '31.19% (1_150 / 3_687)', 'remaining time': '21:16:15', 'throughput': '8413.78', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:35:35,053 >> {'loss': 0.429, 'grad_norm': 14.300618171691895, 'learning_rate': 8.413450466261691e-07, 'epoch': 0.040954705722809875, 'num_input_tokens_seen': 2413821952, 'completed': '31.22% (1_151 / 3_687)', 'remaining time': '21:16:17', 'throughput': '8163.95', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:36:04,990 >> {'loss': 0.3731, 'grad_norm': 14.614937782287598, 'learning_rate': 8.410372696424535e-07, 'epoch': 0.04122592893951722, 'num_input_tokens_seen': 2415919104, 'completed': '31.24% (1_152 / 3_687)', 'remaining time': '21:15:42', 'throughput': '8756.49', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:36:35,431 >> {'loss': 0.4024, 'grad_norm': 12.267914772033691, 'learning_rate': 8.40729258443018e-07, 'epoch': 0.04149715215622457, 'num_input_tokens_seen': 2418016256, 'completed': '31.27% (1_153 / 3_687)', 'remaining time': '21:15:16', 'throughput': '8611.58', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:37:04,253 >> {'loss': 0.3267, 'grad_norm': 12.720690727233887, 'learning_rate': 8.404210132757385e-07, 'epoch': 0.041768375372931925, 'num_input_tokens_seen': 2420113408, 'completed': '31.30% (1_154 / 3_687)', 'remaining time': '21:14:23', 'throughput': '9095.26', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:37:33,986 >> {'loss': 0.5473, 'grad_norm': 15.737774848937988, 'learning_rate': 8.401125343886787e-07, 'epoch': 0.04203959858963927, 'num_input_tokens_seen': 2422210560, 'completed': '31.33% (1_155 / 3_687)', 'remaining time': '21:13:46', 'throughput': '8816.37', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:38:04,141 >> {'loss': 0.3099, 'grad_norm': 9.714899063110352, 'learning_rate': 8.398038220300908e-07, 'epoch': 0.04231082180634662, 'num_input_tokens_seen': 2424307712, 'completed': '31.35% (1_156 / 3_687)', 'remaining time': '21:13:15', 'throughput': '8693.38', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:38:29,843 >> {'loss': 0.7204, 'grad_norm': 18.72726058959961, 'learning_rate': 8.39494876448415e-07, 'epoch': 0.042582045023053974, 'num_input_tokens_seen': 2426404864, 'completed': '31.38% (1_157 / 3_687)', 'remaining time': '21:11:33', 'throughput': '10199.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:38:59,084 >> {'loss': 0.8302, 'grad_norm': 19.449329376220703, 'learning_rate': 8.391856978922785e-07, 'epoch': 0.04285326823976132, 'num_input_tokens_seen': 2428502016, 'completed': '31.41% (1_158 / 3_687)', 'remaining time': '21:10:48', 'throughput': '8964.92', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:39:29,631 >> {'loss': 0.2353, 'grad_norm': 12.25051212310791, 'learning_rate': 8.38876286610497e-07, 'epoch': 0.043124491456468676, 'num_input_tokens_seen': 2430599168, 'completed': '31.43% (1_159 / 3_687)', 'remaining time': '21:10:24', 'throughput': '8581.93', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:39:59,479 >> {'loss': 0.6826, 'grad_norm': 17.40892791748047, 'learning_rate': 8.385666428520723e-07, 'epoch': 0.043395714673176024, 'num_input_tokens_seen': 2432696320, 'completed': '31.46% (1_160 / 3_687)', 'remaining time': '21:09:49', 'throughput': '8782.55', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:40:30,946 >> {'loss': 0.694, 'grad_norm': 20.962051391601562, 'learning_rate': 8.382567668661943e-07, 'epoch': 0.04366693788988337, 'num_input_tokens_seen': 2434793472, 'completed': '31.49% (1_161 / 3_687)', 'remaining time': '21:09:40', 'throughput': '8330.68', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:41:00,777 >> {'loss': 0.6455, 'grad_norm': 17.564403533935547, 'learning_rate': 8.379466589022393e-07, 'epoch': 0.043938161106590726, 'num_input_tokens_seen': 2436890624, 'completed': '31.52% (1_162 / 3_687)', 'remaining time': '21:09:04', 'throughput': '8787.74', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:41:30,832 >> {'loss': 0.5219, 'grad_norm': 16.586353302001953, 'learning_rate': 8.376363192097703e-07, 'epoch': 0.04420938432329807, 'num_input_tokens_seen': 2438987776, 'completed': '31.54% (1_163 / 3_687)', 'remaining time': '21:08:33', 'throughput': '8722.10', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:42:02,404 >> {'loss': 0.6066, 'grad_norm': 20.66847801208496, 'learning_rate': 8.37325748038537e-07, 'epoch': 0.04448060754000543, 'num_input_tokens_seen': 2441084928, 'completed': '31.57% (1_164 / 3_687)', 'remaining time': '21:08:24', 'throughput': '8303.05', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:42:31,402 >> {'loss': 0.4234, 'grad_norm': 17.191225051879883, 'learning_rate': 8.370149456384754e-07, 'epoch': 0.044751830756712775, 'num_input_tokens_seen': 2443182080, 'completed': '31.60% (1_165 / 3_687)', 'remaining time': '21:07:36', 'throughput': '9040.14', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:43:02,305 >> {'loss': 0.3087, 'grad_norm': 13.013045310974121, 'learning_rate': 8.36703912259707e-07, 'epoch': 0.04502305397342012, 'num_input_tokens_seen': 2445279232, 'completed': '31.62% (1_166 / 3_687)', 'remaining time': '21:07:17', 'throughput': '8482.82', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:43:31,344 >> {'loss': 0.7541, 'grad_norm': 22.083459854125977, 'learning_rate': 8.363926481525402e-07, 'epoch': 0.04529427719012748, 'num_input_tokens_seen': 2447376384, 'completed': '31.65% (1_167 / 3_687)', 'remaining time': '21:06:30', 'throughput': '9027.19', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:44:01,312 >> {'loss': 0.4852, 'grad_norm': 14.312165260314941, 'learning_rate': 8.360811535674682e-07, 'epoch': 0.045565500406834825, 'num_input_tokens_seen': 2449473536, 'completed': '31.68% (1_168 / 3_687)', 'remaining time': '21:05:57', 'throughput': '8747.58', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:44:32,103 >> {'loss': 0.7828, 'grad_norm': 20.598003387451172, 'learning_rate': 8.357694287551698e-07, 'epoch': 0.04583672362354217, 'num_input_tokens_seen': 2451570688, 'completed': '31.71% (1_169 / 3_687)', 'remaining time': '21:05:37', 'throughput': '8513.53', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:45:03,402 >> {'loss': 0.4868, 'grad_norm': 19.325916290283203, 'learning_rate': 8.354574739665096e-07, 'epoch': 0.04610794684024953, 'num_input_tokens_seen': 2453667840, 'completed': '31.73% (1_170 / 3_687)', 'remaining time': '21:05:23', 'throughput': '8375.58', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:45:37,007 >> {'loss': 0.3321, 'grad_norm': 12.34057903289795, 'learning_rate': 8.351452894525368e-07, 'epoch': 0.046379170056956874, 'num_input_tokens_seen': 2455764992, 'completed': '31.76% (1_171 / 3_687)', 'remaining time': '21:05:44', 'throughput': '7800.62', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:46:06,669 >> {'loss': 0.4123, 'grad_norm': 12.274550437927246, 'learning_rate': 8.348328754644855e-07, 'epoch': 0.04665039327366423, 'num_input_tokens_seen': 2457862144, 'completed': '31.79% (1_172 / 3_687)', 'remaining time': '21:05:06', 'throughput': '8837.83', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:46:35,978 >> {'loss': 0.5802, 'grad_norm': 14.192599296569824, 'learning_rate': 8.34520232253775e-07, 'epoch': 0.046921616490371576, 'num_input_tokens_seen': 2459959296, 'completed': '31.81% (1_173 / 3_687)', 'remaining time': '21:04:23', 'throughput': '8944.03', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:47:07,034 >> {'loss': 0.6071, 'grad_norm': 20.330703735351562, 'learning_rate': 8.342073600720082e-07, 'epoch': 0.047192839707078924, 'num_input_tokens_seen': 2462056448, 'completed': '31.84% (1_174 / 3_687)', 'remaining time': '21:04:06', 'throughput': '8441.11', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:47:43,424 >> {'loss': 0.4686, 'grad_norm': 16.746747970581055, 'learning_rate': 8.33894259170973e-07, 'epoch': 0.04746406292378628, 'num_input_tokens_seen': 2464153600, 'completed': '31.87% (1_175 / 3_687)', 'remaining time': '21:05:05', 'throughput': '7203.77', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:48:15,167 >> {'loss': 0.329, 'grad_norm': 13.190831184387207, 'learning_rate': 8.335809298026409e-07, 'epoch': 0.047735286140493625, 'num_input_tokens_seen': 2466250752, 'completed': '31.90% (1_176 / 3_687)', 'remaining time': '21:04:56', 'throughput': '8258.36', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:48:47,268 >> {'loss': 0.396, 'grad_norm': 14.298337936401367, 'learning_rate': 8.332673722191677e-07, 'epoch': 0.04800650935720097, 'num_input_tokens_seen': 2468347904, 'completed': '31.92% (1_177 / 3_687)', 'remaining time': '21:04:53', 'throughput': '8166.12', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:49:17,731 >> {'loss': 0.4024, 'grad_norm': 14.075898170471191, 'learning_rate': 8.329535866728922e-07, 'epoch': 0.04827773257390833, 'num_input_tokens_seen': 2470445056, 'completed': '31.95% (1_178 / 3_687)', 'remaining time': '21:04:26', 'throughput': '8605.22', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:49:49,409 >> {'loss': 0.6025, 'grad_norm': 15.5481595993042, 'learning_rate': 8.326395734163375e-07, 'epoch': 0.048548955790615675, 'num_input_tokens_seen': 2472542208, 'completed': '31.98% (1_179 / 3_687)', 'remaining time': '21:04:15', 'throughput': '8275.43', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:50:24,915 >> {'loss': 0.3926, 'grad_norm': 12.739563941955566, 'learning_rate': 8.323253327022094e-07, 'epoch': 0.04882017900732303, 'num_input_tokens_seen': 2474639360, 'completed': '32.00% (1_180 / 3_687)', 'remaining time': '21:04:59', 'throughput': '7383.12', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:50:58,557 >> {'loss': 0.3176, 'grad_norm': 10.208687782287598, 'learning_rate': 8.320108647833967e-07, 'epoch': 0.04909140222403038, 'num_input_tokens_seen': 2476736512, 'completed': '32.03% (1_181 / 3_687)', 'remaining time': '21:05:15', 'throughput': '7792.19', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:51:29,217 >> {'loss': 0.247, 'grad_norm': 9.83712387084961, 'learning_rate': 8.316961699129714e-07, 'epoch': 0.049362625440737724, 'num_input_tokens_seen': 2478833664, 'completed': '32.06% (1_182 / 3_687)', 'remaining time': '21:04:50', 'throughput': '8549.83', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:52:01,476 >> {'loss': 0.7097, 'grad_norm': 19.233394622802734, 'learning_rate': 8.313812483441879e-07, 'epoch': 0.04963384865744508, 'num_input_tokens_seen': 2480930816, 'completed': '32.09% (1_183 / 3_687)', 'remaining time': '21:04:46', 'throughput': '8126.22', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:52:30,419 >> {'loss': 0.33, 'grad_norm': 12.595343589782715, 'learning_rate': 8.310661003304829e-07, 'epoch': 0.049905071874152426, 'num_input_tokens_seen': 2483027968, 'completed': '32.11% (1_184 / 3_687)', 'remaining time': '21:03:57', 'throughput': '9057.21', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:52:59,321 >> {'loss': 0.4683, 'grad_norm': 14.636686325073242, 'learning_rate': 8.30750726125476e-07, 'epoch': 0.05017629509085978, 'num_input_tokens_seen': 2485125120, 'completed': '32.14% (1_185 / 3_687)', 'remaining time': '21:03:08', 'throughput': '9070.10', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:53:27,887 >> {'loss': 0.4655, 'grad_norm': 13.301069259643555, 'learning_rate': 8.304351259829678e-07, 'epoch': 0.05044751830756713, 'num_input_tokens_seen': 2487222272, 'completed': '32.17% (1_186 / 3_687)', 'remaining time': '21:02:15', 'throughput': '9177.00', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:53:56,613 >> {'loss': 0.3759, 'grad_norm': 13.354597091674805, 'learning_rate': 8.301193001569418e-07, 'epoch': 0.050718741524274476, 'num_input_tokens_seen': 2489319424, 'completed': '32.19% (1_187 / 3_687)', 'remaining time': '21:01:24', 'throughput': '9125.55', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:54:27,565 >> {'loss': 0.3448, 'grad_norm': 12.58713150024414, 'learning_rate': 8.298032489015623e-07, 'epoch': 0.05098996474098183, 'num_input_tokens_seen': 2491416576, 'completed': '32.22% (1_188 / 3_687)', 'remaining time': '21:01:02', 'throughput': '8469.51', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:54:56,377 >> {'loss': 0.586, 'grad_norm': 19.054805755615234, 'learning_rate': 8.294869724711752e-07, 'epoch': 0.05126118795768918, 'num_input_tokens_seen': 2493513728, 'completed': '32.25% (1_189 / 3_687)', 'remaining time': '21:00:13', 'throughput': '9098.19', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:55:27,239 >> {'loss': 0.3833, 'grad_norm': 11.936301231384277, 'learning_rate': 8.291704711203082e-07, 'epoch': 0.051532411174396525, 'num_input_tokens_seen': 2495610880, 'completed': '32.28% (1_190 / 3_687)', 'remaining time': '20:59:50', 'throughput': '8494.21', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:55:59,718 >> {'loss': 0.4227, 'grad_norm': 18.23284912109375, 'learning_rate': 8.288537451036691e-07, 'epoch': 0.05180363439110388, 'num_input_tokens_seen': 2497708032, 'completed': '32.30% (1_191 / 3_687)', 'remaining time': '20:59:49', 'throughput': '8071.06', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:56:31,070 >> {'loss': 0.3232, 'grad_norm': 14.435386657714844, 'learning_rate': 8.28536794676147e-07, 'epoch': 0.05207485760781123, 'num_input_tokens_seen': 2499805184, 'completed': '32.33% (1_192 / 3_687)', 'remaining time': '20:59:32', 'throughput': '8361.41', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:57:04,242 >> {'loss': 0.4033, 'grad_norm': 13.897747039794922, 'learning_rate': 8.282196200928119e-07, 'epoch': 0.05234608082451858, 'num_input_tokens_seen': 2501902336, 'completed': '32.36% (1_193 / 3_687)', 'remaining time': '20:59:39', 'throughput': '7902.56', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:57:34,381 >> {'loss': 0.3174, 'grad_norm': 11.174321174621582, 'learning_rate': 8.279022216089135e-07, 'epoch': 0.05261730404122593, 'num_input_tokens_seen': 2503999488, 'completed': '32.38% (1_194 / 3_687)', 'remaining time': '20:59:07', 'throughput': '8698.00', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:58:05,139 >> {'loss': 0.4879, 'grad_norm': 15.450333595275879, 'learning_rate': 8.275845994798821e-07, 'epoch': 0.05288852725793328, 'num_input_tokens_seen': 2506096640, 'completed': '32.41% (1_195 / 3_687)', 'remaining time': '20:58:42', 'throughput': '8522.65', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:58:34,400 >> {'loss': 0.3927, 'grad_norm': 12.975201606750488, 'learning_rate': 8.272667539613281e-07, 'epoch': 0.05315975047464063, 'num_input_tokens_seen': 2508193792, 'completed': '32.44% (1_196 / 3_687)', 'remaining time': '20:57:59', 'throughput': '8958.83', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:59:05,682 >> {'loss': 0.6859, 'grad_norm': 20.552017211914062, 'learning_rate': 8.26948685309041e-07, 'epoch': 0.05343097369134798, 'num_input_tokens_seen': 2510290944, 'completed': '32.47% (1_197 / 3_687)', 'remaining time': '20:57:41', 'throughput': '8380.07', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 03:59:37,398 >> {'loss': 0.4479, 'grad_norm': 14.239463806152344, 'learning_rate': 8.266303937789908e-07, 'epoch': 0.05370219690805533, 'num_input_tokens_seen': 2512388096, 'completed': '32.49% (1_198 / 3_687)', 'remaining time': '20:57:28', 'throughput': '8265.23', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:00:06,751 >> {'loss': 0.5931, 'grad_norm': 13.604122161865234, 'learning_rate': 8.263118796273263e-07, 'epoch': 0.05397342012476268, 'num_input_tokens_seen': 2514485248, 'completed': '32.52% (1_199 / 3_687)', 'remaining time': '20:56:46', 'throughput': '8930.89', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:00:38,358 >> {'loss': 0.4736, 'grad_norm': 22.870668411254883, 'learning_rate': 8.259931431103754e-07, 'epoch': 0.05424464334147003, 'num_input_tokens_seen': 2516582400, 'completed': '32.55% (1_200 / 3_687)', 'remaining time': '20:56:32', 'throughput': '8293.69', 'gpu_mem_free': '5581MB'}
+/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+[INFO|trainer.py:3503] 2025-01-02 04:01:02,359 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200
+[INFO|configuration_utils.py:472] 2025-01-02 04:01:02,365 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/config.json
+[INFO|configuration_utils.py:807] 2025-01-02 04:01:02,366 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/generation_config.json
+[INFO|modeling_utils.py:2807] 2025-01-02 04:02:02,595 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/model.safetensors.index.json.
+[INFO|tokenization_utils_base.py:2684] 2025-01-02 04:02:02,600 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2693] 2025-01-02 04:02:02,600 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1200/special_tokens_map.json
+/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+[WARNING|trainer.py:868] 2025-01-02 04:05:41,083 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 2400, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1}
+01/02/2025 04:05:41 - WARNING - streaming.base.dataset - Because `shuffle_block_size` was not specified, it will default to max(4_000_000 // num_canonical_nodes, 1 << 18) if num_canonical_nodes is not None, otherwise 262144. Prior to Streaming v0.7.0, `shuffle_block_size` defaulted to 262144.
+/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context:  # type: ignore[attr-defined]
+[INFO|trainer.py:175] 2025-01-02 04:06:13,247 >> {'loss': 0.5313, 'grad_norm': 15.544678688049316, 'learning_rate': 8.256741844846452e-07, 'epoch': 0.05451586655817738, 'num_input_tokens_seen': 2518679552, 'completed': '32.57% (1_201 / 3_687)', 'remaining time': '21:58:49', 'throughput': '782.78', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:06:47,174 >> {'loss': 0.3772, 'grad_norm': 12.727087020874023, 'learning_rate': 8.253550040068216e-07, 'epoch': 0.05478708977488473, 'num_input_tokens_seen': 2520776704, 'completed': '32.60% (1_202 / 3_687)', 'remaining time': '21:58:43', 'throughput': '7726.61', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:07:20,530 >> {'loss': 0.4334, 'grad_norm': 14.271738052368164, 'learning_rate': 8.250356019337688e-07, 'epoch': 0.05505831299159208, 'num_input_tokens_seen': 2522873856, 'completed': '32.63% (1_203 / 3_687)', 'remaining time': '21:58:29', 'throughput': '7859.06', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:07:48,568 >> {'loss': 0.939, 'grad_norm': 20.528383255004883, 'learning_rate': 8.247159785225295e-07, 'epoch': 0.05532953620829943, 'num_input_tokens_seen': 2524971008, 'completed': '32.66% (1_204 / 3_687)', 'remaining time': '21:57:11', 'throughput': '9349.75', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:08:16,023 >> {'loss': 0.7866, 'grad_norm': 21.59691619873047, 'learning_rate': 8.243961340303245e-07, 'epoch': 0.05560075942500678, 'num_input_tokens_seen': 2527068160, 'completed': '32.68% (1_205 / 3_687)', 'remaining time': '21:55:46', 'throughput': '9547.89', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:08:46,029 >> {'loss': 0.2154, 'grad_norm': 8.262271881103516, 'learning_rate': 8.240760687145521e-07, 'epoch': 0.055871982641714134, 'num_input_tokens_seen': 2529165312, 'completed': '32.71% (1_206 / 3_687)', 'remaining time': '21:54:53', 'throughput': '8736.56', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:09:15,965 >> {'loss': 0.6242, 'grad_norm': 16.18045425415039, 'learning_rate': 8.237557828327891e-07, 'epoch': 0.05614320585842148, 'num_input_tokens_seen': 2531262464, 'completed': '32.74% (1_207 / 3_687)', 'remaining time': '21:53:59', 'throughput': '8756.70', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:09:47,480 >> {'loss': 0.7028, 'grad_norm': 19.58281707763672, 'learning_rate': 8.234352766427894e-07, 'epoch': 0.05641442907512883, 'num_input_tokens_seen': 2533359616, 'completed': '32.76% (1_208 / 3_687)', 'remaining time': '21:53:24', 'throughput': '8318.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:10:16,664 >> {'loss': 0.7435, 'grad_norm': 19.143722534179688, 'learning_rate': 8.231145504024838e-07, 'epoch': 0.05668565229183618, 'num_input_tokens_seen': 2535456768, 'completed': '32.79% (1_209 / 3_687)', 'remaining time': '21:52:21', 'throughput': '8982.56', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:10:46,833 >> {'loss': 0.4029, 'grad_norm': 10.882222175598145, 'learning_rate': 8.22793604369981e-07, 'epoch': 0.05695687550854353, 'num_input_tokens_seen': 2537553920, 'completed': '32.82% (1_210 / 3_687)', 'remaining time': '21:51:30', 'throughput': '8689.04', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:11:17,357 >> {'loss': 0.3975, 'grad_norm': 13.754764556884766, 'learning_rate': 8.224724388035659e-07, 'epoch': 0.05722809872525088, 'num_input_tokens_seen': 2539651072, 'completed': '32.85% (1_211 / 3_687)', 'remaining time': '21:50:44', 'throughput': '8588.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:11:47,053 >> {'loss': 0.5442, 'grad_norm': 17.92790412902832, 'learning_rate': 8.221510539617003e-07, 'epoch': 0.05749932194195823, 'num_input_tokens_seen': 2541748224, 'completed': '32.87% (1_212 / 3_687)', 'remaining time': '21:49:48', 'throughput': '8827.71', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:12:17,947 >> {'loss': 0.3787, 'grad_norm': 12.144906997680664, 'learning_rate': 8.218294501030226e-07, 'epoch': 0.05777054515866558, 'num_input_tokens_seen': 2543845376, 'completed': '32.90% (1_213 / 3_687)', 'remaining time': '21:49:06', 'throughput': '8485.10', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:12:49,727 >> {'loss': 0.5476, 'grad_norm': 15.397769927978516, 'learning_rate': 8.215076274863476e-07, 'epoch': 0.058041768375372935, 'num_input_tokens_seen': 2545942528, 'completed': '32.93% (1_214 / 3_687)', 'remaining time': '21:48:35', 'throughput': '8248.90', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:13:21,159 >> {'loss': 0.4261, 'grad_norm': 13.692743301391602, 'learning_rate': 8.211855863706654e-07, 'epoch': 0.05831299159208028, 'num_input_tokens_seen': 2548039680, 'completed': '32.95% (1_215 / 3_687)', 'remaining time': '21:47:59', 'throughput': '8339.91', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:13:51,951 >> {'loss': 0.5572, 'grad_norm': 15.524927139282227, 'learning_rate': 8.208633270151426e-07, 'epoch': 0.05858421480878763, 'num_input_tokens_seen': 2550136832, 'completed': '32.98% (1_216 / 3_687)', 'remaining time': '21:47:17', 'throughput': '8513.34', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:14:22,788 >> {'loss': 0.6069, 'grad_norm': 21.096981048583984, 'learning_rate': 8.205408496791216e-07, 'epoch': 0.058855438025494984, 'num_input_tokens_seen': 2552233984, 'completed': '33.01% (1_217 / 3_687)', 'remaining time': '21:46:35', 'throughput': '8500.93', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:14:56,803 >> {'loss': 0.3785, 'grad_norm': 12.139968872070312, 'learning_rate': 8.202181546221193e-07, 'epoch': 0.05912666124220233, 'num_input_tokens_seen': 2554331136, 'completed': '33.03% (1_218 / 3_687)', 'remaining time': '21:46:29', 'throughput': '7706.67', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:15:24,489 >> {'loss': 0.7316, 'grad_norm': 18.615991592407227, 'learning_rate': 8.19895242103829e-07, 'epoch': 0.059397884458909686, 'num_input_tokens_seen': 2556428288, 'completed': '33.06% (1_219 / 3_687)', 'remaining time': '21:45:11', 'throughput': '9468.75', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:15:53,152 >> {'loss': 0.7, 'grad_norm': 20.739112854003906, 'learning_rate': 8.19572112384118e-07, 'epoch': 0.059669107675617034, 'num_input_tokens_seen': 2558525440, 'completed': '33.09% (1_220 / 3_687)', 'remaining time': '21:44:05', 'throughput': '9145.54', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:16:24,038 >> {'loss': 0.3302, 'grad_norm': 13.558570861816406, 'learning_rate': 8.192487657230288e-07, 'epoch': 0.05994033089232438, 'num_input_tokens_seen': 2560622592, 'completed': '33.12% (1_221 / 3_687)', 'remaining time': '21:43:24', 'throughput': '8487.67', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:16:57,408 >> {'loss': 0.3308, 'grad_norm': 12.751009941101074, 'learning_rate': 8.18925202380779e-07, 'epoch': 0.060211554109031735, 'num_input_tokens_seen': 2562719744, 'completed': '33.14% (1_222 / 3_687)', 'remaining time': '21:43:11', 'throughput': '7855.47', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:17:29,981 >> {'loss': 0.5112, 'grad_norm': 14.548094749450684, 'learning_rate': 8.186014226177594e-07, 'epoch': 0.06048277732573908, 'num_input_tokens_seen': 2564816896, 'completed': '33.17% (1_223 / 3_687)', 'remaining time': '21:42:48', 'throughput': '8048.06', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:17:59,436 >> {'loss': 0.5099, 'grad_norm': 13.776844024658203, 'learning_rate': 8.18277426694536e-07, 'epoch': 0.06075400054244643, 'num_input_tokens_seen': 2566914048, 'completed': '33.20% (1_224 / 3_687)', 'remaining time': '21:41:52', 'throughput': '8899.60', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:18:29,725 >> {'loss': 0.5184, 'grad_norm': 14.455738067626953, 'learning_rate': 8.179532148718483e-07, 'epoch': 0.061025223759153785, 'num_input_tokens_seen': 2569011200, 'completed': '33.22% (1_225 / 3_687)', 'remaining time': '21:41:04', 'throughput': '8654.78', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:19:00,435 >> {'loss': 0.6313, 'grad_norm': 16.7779598236084, 'learning_rate': 8.176287874106097e-07, 'epoch': 0.06129644697586113, 'num_input_tokens_seen': 2571108352, 'completed': '33.25% (1_226 / 3_687)', 'remaining time': '21:40:22', 'throughput': '8536.12', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:19:27,921 >> {'loss': 0.7062, 'grad_norm': 17.045129776000977, 'learning_rate': 8.173041445719069e-07, 'epoch': 0.06156767019256849, 'num_input_tokens_seen': 2573205504, 'completed': '33.28% (1_227 / 3_687)', 'remaining time': '21:39:04', 'throughput': '9537.51', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:19:59,897 >> {'loss': 0.8444, 'grad_norm': 23.766611099243164, 'learning_rate': 8.169792866170003e-07, 'epoch': 0.061838893409275834, 'num_input_tokens_seen': 2575302656, 'completed': '33.31% (1_228 / 3_687)', 'remaining time': '21:38:36', 'throughput': '8198.14', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:20:29,050 >> {'loss': 0.5219, 'grad_norm': 13.02985668182373, 'learning_rate': 8.166542138073232e-07, 'epoch': 0.06211011662598318, 'num_input_tokens_seen': 2577399808, 'completed': '33.33% (1_229 / 3_687)', 'remaining time': '21:37:37', 'throughput': '8991.95', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:21:01,220 >> {'loss': 0.6017, 'grad_norm': 15.076738357543945, 'learning_rate': 8.163289264044817e-07, 'epoch': 0.062381339842690536, 'num_input_tokens_seen': 2579496960, 'completed': '33.36% (1_230 / 3_687)', 'remaining time': '21:37:11', 'throughput': '8148.74', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:21:31,907 >> {'loss': 0.5749, 'grad_norm': 15.141392707824707, 'learning_rate': 8.160034246702548e-07, 'epoch': 0.06265256305939788, 'num_input_tokens_seen': 2581594112, 'completed': '33.39% (1_231 / 3_687)', 'remaining time': '21:36:28', 'throughput': '8542.41', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:22:02,871 >> {'loss': 0.4558, 'grad_norm': 13.555374145507812, 'learning_rate': 8.156777088665939e-07, 'epoch': 0.06292378627610523, 'num_input_tokens_seen': 2583691264, 'completed': '33.41% (1_232 / 3_687)', 'remaining time': '21:35:49', 'throughput': '8466.22', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:22:33,348 >> {'loss': 0.3044, 'grad_norm': 13.061405181884766, 'learning_rate': 8.153517792556226e-07, 'epoch': 0.06319500949281258, 'num_input_tokens_seen': 2585788416, 'completed': '33.44% (1_233 / 3_687)', 'remaining time': '21:35:05', 'throughput': '8601.22', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:23:01,379 >> {'loss': 0.6267, 'grad_norm': 17.51995277404785, 'learning_rate': 8.15025636099637e-07, 'epoch': 0.06346623270951994, 'num_input_tokens_seen': 2587885568, 'completed': '33.47% (1_234 / 3_687)', 'remaining time': '21:33:55', 'throughput': '9352.11', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:23:34,089 >> {'loss': 0.3617, 'grad_norm': 13.34430980682373, 'learning_rate': 8.146992796611042e-07, 'epoch': 0.06373745592622729, 'num_input_tokens_seen': 2589982720, 'completed': '33.50% (1_235 / 3_687)', 'remaining time': '21:33:35', 'throughput': '8014.23', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:24:00,555 >> {'loss': 0.8845, 'grad_norm': 18.68717384338379, 'learning_rate': 8.143727102026638e-07, 'epoch': 0.06400867914293464, 'num_input_tokens_seen': 2592079872, 'completed': '33.52% (1_236 / 3_687)', 'remaining time': '21:32:09', 'throughput': '9904.72', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:24:30,910 >> {'loss': 0.3102, 'grad_norm': 13.216950416564941, 'learning_rate': 8.140459279871264e-07, 'epoch': 0.06427990235964198, 'num_input_tokens_seen': 2594177024, 'completed': '33.55% (1_237 / 3_687)', 'remaining time': '21:31:24', 'throughput': '8636.22', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:25:00,875 >> {'loss': 0.7681, 'grad_norm': 22.348615646362305, 'learning_rate': 8.137189332774738e-07, 'epoch': 0.06455112557634933, 'num_input_tokens_seen': 2596274176, 'completed': '33.58% (1_238 / 3_687)', 'remaining time': '21:30:36', 'throughput': '8748.03', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:25:30,514 >> {'loss': 0.3414, 'grad_norm': 12.911705017089844, 'learning_rate': 8.133917263368589e-07, 'epoch': 0.06482234879305669, 'num_input_tokens_seen': 2598371328, 'completed': '33.60% (1_239 / 3_687)', 'remaining time': '21:29:44', 'throughput': '8844.61', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:26:01,069 >> {'loss': 0.4748, 'grad_norm': 19.478103637695312, 'learning_rate': 8.130643074286056e-07, 'epoch': 0.06509357200976404, 'num_input_tokens_seen': 2600468480, 'completed': '33.63% (1_240 / 3_687)', 'remaining time': '21:29:01', 'throughput': '8579.34', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:26:37,315 >> {'loss': 0.4883, 'grad_norm': 16.46529197692871, 'learning_rate': 8.127366768162077e-07, 'epoch': 0.06536479522647139, 'num_input_tokens_seen': 2602565632, 'completed': '33.66% (1_241 / 3_687)', 'remaining time': '21:29:17', 'throughput': '7232.36', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:27:09,210 >> {'loss': 0.4281, 'grad_norm': 13.854009628295898, 'learning_rate': 8.124088347633304e-07, 'epoch': 0.06563601844317873, 'num_input_tokens_seen': 2604662784, 'completed': '33.69% (1_242 / 3_687)', 'remaining time': '21:28:48', 'throughput': '8219.10', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:27:39,418 >> {'loss': 0.4065, 'grad_norm': 12.586260795593262, 'learning_rate': 8.120807815338083e-07, 'epoch': 0.06590724165988608, 'num_input_tokens_seen': 2606759936, 'completed': '33.71% (1_243 / 3_687)', 'remaining time': '21:28:02', 'throughput': '8677.89', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:28:12,754 >> {'loss': 0.3784, 'grad_norm': 12.778905868530273, 'learning_rate': 8.11752517391646e-07, 'epoch': 0.06617846487659344, 'num_input_tokens_seen': 2608857088, 'completed': '33.74% (1_244 / 3_687)', 'remaining time': '21:27:47', 'throughput': '7863.80', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:28:43,792 >> {'loss': 0.3571, 'grad_norm': 13.905147552490234, 'learning_rate': 8.114240426010183e-07, 'epoch': 0.06644968809330079, 'num_input_tokens_seen': 2610954240, 'completed': '33.77% (1_245 / 3_687)', 'remaining time': '21:27:10', 'throughput': '8445.80', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:29:13,042 >> {'loss': 0.6425, 'grad_norm': 18.284894943237305, 'learning_rate': 8.11095357426269e-07, 'epoch': 0.06672091131000814, 'num_input_tokens_seen': 2613051392, 'completed': '33.79% (1_246 / 3_687)', 'remaining time': '21:26:15', 'throughput': '8962.34', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:29:42,766 >> {'loss': 0.5305, 'grad_norm': 14.604440689086914, 'learning_rate': 8.107664621319113e-07, 'epoch': 0.06699213452671549, 'num_input_tokens_seen': 2615148544, 'completed': '33.82% (1_247 / 3_687)', 'remaining time': '21:25:24', 'throughput': '8819.15', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:30:12,192 >> {'loss': 0.4138, 'grad_norm': 14.840185165405273, 'learning_rate': 8.10437356982628e-07, 'epoch': 0.06726335774342283, 'num_input_tokens_seen': 2617245696, 'completed': '33.85% (1_248 / 3_687)', 'remaining time': '21:24:31', 'throughput': '8908.50', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:30:46,121 >> {'loss': 0.466, 'grad_norm': 13.899113655090332, 'learning_rate': 8.1010804224327e-07, 'epoch': 0.06753458096013018, 'num_input_tokens_seen': 2619342848, 'completed': '33.88% (1_249 / 3_687)', 'remaining time': '21:24:23', 'throughput': '7726.36', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:31:15,595 >> {'loss': 0.3711, 'grad_norm': 12.415678977966309, 'learning_rate': 8.097785181788574e-07, 'epoch': 0.06780580417683754, 'num_input_tokens_seen': 2621440000, 'completed': '33.90% (1_250 / 3_687)', 'remaining time': '21:23:30', 'throughput': '8894.03', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:31:46,618 >> {'loss': 0.3472, 'grad_norm': 12.70449161529541, 'learning_rate': 8.09448785054579e-07, 'epoch': 0.06807702739354489, 'num_input_tokens_seen': 2623537152, 'completed': '33.93% (1_251 / 3_687)', 'remaining time': '21:22:53', 'throughput': '8449.95', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:32:14,774 >> {'loss': 0.9622, 'grad_norm': 20.355985641479492, 'learning_rate': 8.091188431357908e-07, 'epoch': 0.06834825061025224, 'num_input_tokens_seen': 2625634304, 'completed': '33.96% (1_252 / 3_687)', 'remaining time': '21:21:48', 'throughput': '9310.32', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:32:44,245 >> {'loss': 0.91, 'grad_norm': 19.507715225219727, 'learning_rate': 8.087886926880181e-07, 'epoch': 0.06861947382695958, 'num_input_tokens_seen': 2627731456, 'completed': '33.98% (1_253 / 3_687)', 'remaining time': '21:20:56', 'throughput': '8895.08', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:33:17,264 >> {'loss': 0.4888, 'grad_norm': 13.477721214294434, 'learning_rate': 8.084583339769531e-07, 'epoch': 0.06889069704366693, 'num_input_tokens_seen': 2629828608, 'completed': '34.01% (1_254 / 3_687)', 'remaining time': '21:20:38', 'throughput': '7939.17', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:33:47,963 >> {'loss': 0.438, 'grad_norm': 15.250035285949707, 'learning_rate': 8.081277672684557e-07, 'epoch': 0.0691619202603743, 'num_input_tokens_seen': 2631925760, 'completed': '34.04% (1_255 / 3_687)', 'remaining time': '21:19:58', 'throughput': '8539.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:34:15,764 >> {'loss': 0.5853, 'grad_norm': 13.72596263885498, 'learning_rate': 8.077969928285541e-07, 'epoch': 0.06943314347708164, 'num_input_tokens_seen': 2634022912, 'completed': '34.07% (1_256 / 3_687)', 'remaining time': '21:18:51', 'throughput': '9429.42', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:34:46,720 >> {'loss': 0.456, 'grad_norm': 23.236064910888672, 'learning_rate': 8.074660109234424e-07, 'epoch': 0.06970436669378899, 'num_input_tokens_seen': 2636120064, 'completed': '34.09% (1_257 / 3_687)', 'remaining time': '21:18:14', 'throughput': '8468.28', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:35:16,377 >> {'loss': 0.5927, 'grad_norm': 19.698314666748047, 'learning_rate': 8.071348218194823e-07, 'epoch': 0.06997558991049634, 'num_input_tokens_seen': 2638217216, 'completed': '34.12% (1_258 / 3_687)', 'remaining time': '21:17:24', 'throughput': '8839.13', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:35:46,815 >> {'loss': 0.7272, 'grad_norm': 17.194185256958008, 'learning_rate': 8.068034257832026e-07, 'epoch': 0.07024681312720368, 'num_input_tokens_seen': 2640314368, 'completed': '34.15% (1_259 / 3_687)', 'remaining time': '21:16:42', 'throughput': '8612.51', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:36:15,182 >> {'loss': 0.4973, 'grad_norm': 14.79769229888916, 'learning_rate': 8.064718230812976e-07, 'epoch': 0.07051803634391104, 'num_input_tokens_seen': 2642411520, 'completed': '34.17% (1_260 / 3_687)', 'remaining time': '21:15:41', 'throughput': '9241.16', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:36:45,415 >> {'loss': 0.7414, 'grad_norm': 18.805007934570312, 'learning_rate': 8.06140013980629e-07, 'epoch': 0.07078925956061839, 'num_input_tokens_seen': 2644508672, 'completed': '34.20% (1_261 / 3_687)', 'remaining time': '21:14:57', 'throughput': '8670.76', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:37:16,834 >> {'loss': 0.3165, 'grad_norm': 10.652469635009766, 'learning_rate': 8.05807998748224e-07, 'epoch': 0.07106048277732574, 'num_input_tokens_seen': 2646605824, 'completed': '34.23% (1_262 / 3_687)', 'remaining time': '21:14:25', 'throughput': '8343.33', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:37:46,764 >> {'loss': 0.3287, 'grad_norm': 10.750865936279297, 'learning_rate': 8.05475777651276e-07, 'epoch': 0.07133170599403309, 'num_input_tokens_seen': 2648702976, 'completed': '34.26% (1_263 / 3_687)', 'remaining time': '21:13:38', 'throughput': '8758.68', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:38:15,609 >> {'loss': 0.6628, 'grad_norm': 17.767995834350586, 'learning_rate': 8.051433509571435e-07, 'epoch': 0.07160292921074043, 'num_input_tokens_seen': 2650800128, 'completed': '34.28% (1_264 / 3_687)', 'remaining time': '21:12:42', 'throughput': '9087.90', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:38:47,686 >> {'loss': 0.6028, 'grad_norm': 17.3983097076416, 'learning_rate': 8.04810718933351e-07, 'epoch': 0.0718741524274478, 'num_input_tokens_seen': 2652897280, 'completed': '34.31% (1_265 / 3_687)', 'remaining time': '21:12:16', 'throughput': '8172.54', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:39:17,452 >> {'loss': 0.3962, 'grad_norm': 12.742525100708008, 'learning_rate': 8.044778818475884e-07, 'epoch': 0.07214537564415514, 'num_input_tokens_seen': 2654994432, 'completed': '34.34% (1_266 / 3_687)', 'remaining time': '21:11:28', 'throughput': '8806.76', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:39:47,530 >> {'loss': 0.6197, 'grad_norm': 15.34299373626709, 'learning_rate': 8.0414483996771e-07, 'epoch': 0.07241659886086249, 'num_input_tokens_seen': 2657091584, 'completed': '34.36% (1_267 / 3_687)', 'remaining time': '21:10:44', 'throughput': '8715.37', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:40:19,517 >> {'loss': 0.8734, 'grad_norm': 18.275728225708008, 'learning_rate': 8.038115935617355e-07, 'epoch': 0.07268782207756984, 'num_input_tokens_seen': 2659188736, 'completed': '34.39% (1_268 / 3_687)', 'remaining time': '21:10:17', 'throughput': '8195.51', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:40:49,334 >> {'loss': 0.3667, 'grad_norm': 11.38978099822998, 'learning_rate': 8.034781428978484e-07, 'epoch': 0.07295904529427719, 'num_input_tokens_seen': 2661285888, 'completed': '34.42% (1_269 / 3_687)', 'remaining time': '21:09:30', 'throughput': '8791.66', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:41:19,011 >> {'loss': 0.2297, 'grad_norm': 17.417783737182617, 'learning_rate': 8.031444882443976e-07, 'epoch': 0.07323026851098453, 'num_input_tokens_seen': 2663383040, 'completed': '34.45% (1_270 / 3_687)', 'remaining time': '21:08:42', 'throughput': '8833.25', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:41:51,598 >> {'loss': 0.6265, 'grad_norm': 15.738312721252441, 'learning_rate': 8.028106298698957e-07, 'epoch': 0.0735014917276919, 'num_input_tokens_seen': 2665480192, 'completed': '34.47% (1_271 / 3_687)', 'remaining time': '21:08:20', 'throughput': '8044.26', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:42:22,177 >> {'loss': 0.3186, 'grad_norm': 14.270974159240723, 'learning_rate': 8.024765680430188e-07, 'epoch': 0.07377271494439924, 'num_input_tokens_seen': 2667577344, 'completed': '34.50% (1_272 / 3_687)', 'remaining time': '21:07:41', 'throughput': '8572.82', 'gpu_mem_free': '5581MB'}
+[INFO|trainer.py:175] 2025-01-02 04:42:53,382 >> {'loss': 0.5579, 'grad_norm': 13.561492919921875, 'learning_rate': 8.021423030326075e-07, 'epoch': 0.07404393816110659, 'num_input_tokens_seen': 2669674496, 'completed': '34.53% (1_273 / 3_687)', 'remaining time': '21:07:07', 'throughput': '8400.70', 'gpu_mem_free': '5581MB'}

wandb/run-20250102_021927-pw8rud5e/files/requirements.txt

@@ -0,0 +1,244 @@
+Brotli==1.1.0
+GitPython==3.1.43
+Jinja2==3.1.4
+MarkupSafe==3.0.2
+PyJWT==2.10.0
+PyNaCl==1.5.0
+PyYAML==6.0.2
+Pygments==2.18.0
+accelerate==0.32.1
+aiohappyeyeballs==2.4.3
+aiohttp==3.11.2
+aioprometheus==23.12.0
+aiosignal==1.3.1
+annotated-types==0.7.0
+anthropic==0.39.0
+anyio==4.6.2.post1
+argcomplete==3.5.1
+arrow==1.3.0
+asttokens==2.4.1
+async-timeout==5.0.1
+attrs==24.2.0
+autocommand==2.2.2
+azure-core==1.32.0
+azure-identity==1.19.0
+azure-storage-blob==12.24.0
+azure-storage-file-datalake==12.18.0
+backoff==2.2.1
+backports.tarfile==1.2.0
+bcrypt==4.2.0
+blobfile==3.0.0
+boto3==1.35.63
+botocore==1.35.63
+cachetools==5.5.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.4.0
+circuitbreaker==2.0.0
+click==8.1.7
+cloudpickle==3.1.0
+comm==0.2.2
+compressed-tensors==0.8.0
+contourpy==1.3.1
+cramjam==2.9.0
+cryptography==43.0.3
+cycler==0.12.1
+datasets==2.20.0
+datatools==0.1
+debugpy==1.8.11
+decorator==5.1.1
+dill==0.3.8
+diskcache==5.6.3
+distro==1.9.0
+docker-pycreds==0.4.0
+docstring_parser==0.16
+einops==0.8.0
+exceptiongroup==1.2.2
+executing==2.1.0
+fastapi==0.115.5
+filelock==3.16.1
+flash-attn==2.6.1
+fonttools==4.55.0
+frozenlist==1.5.0
+fsspec==2024.5.0
+gguf==0.10.0
+gitdb==4.0.11
+google-api-core==2.23.0
+google-auth==2.36.0
+google-cloud-aiplatform==1.71.1
+google-cloud-bigquery==3.27.0
+google-cloud-core==2.4.1
+google-cloud-resource-manager==1.13.1
+google-cloud-storage==2.10.0
+google-crc32c==1.6.0
+google-resumable-media==2.7.2
+googleapis-common-protos==1.66.0
+gql==3.5.0
+graphql-core==3.2.5
+grpc-google-iam-v1==0.13.1
+grpcio-status==1.62.3
+grpcio==1.68.0
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.27.2
+huggingface-hub==0.26.2
+idna==3.10
+importlib_metadata==8.0.0
+importlib_metadata==8.5.0
+inflect==7.3.1
+interegular==0.3.3
+ipykernel==6.29.5
+ipython==8.18.0
+isodate==0.7.2
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+jedi==0.19.2
+jiter==0.7.1
+jmespath==1.0.1
+jsonschema-specifications==2024.10.1
+jsonschema==4.23.0
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+kiwisolver==1.4.7
+lark==1.2.2
+llvmlite==0.43.0
+lm-format-enforcer==0.10.9
+lxml==5.3.0
+markdown-it-py==3.0.0
+matplotlib-inline==0.1.7
+matplotlib==3.9.2
+mdurl==0.1.2
+more-itertools==10.3.0
+mosaicml-cli==0.5.34
+mosaicml-streaming==0.8.1
+mpmath==1.3.0
+msal-extensions==1.2.0
+msal==1.31.1
+msgpack==1.1.0
+msgspec==0.18.6
+multidict==6.1.0
+multiprocess==0.70.16
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.11.1.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py==12.560.30
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+oci==2.138.1
+openai==1.54.5
+opencv-python-headless==4.10.0.84
+orjson==3.10.11
+outlines==0.0.46
+packaging==24.1
+packaging==24.2
+pandas==2.2.1
+paramiko==3.5.0
+parso==0.8.4
+partial-json-parser==0.2.1.1.post4
+pexpect==4.9.0
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+platformdirs==4.3.6
+portalocker==2.10.1
+prometheus-fastapi-instrumentator==7.0.0
+prometheus_client==0.21.0
+prompt-toolkit==3.0.36
+propcache==0.2.0
+proto-plus==1.25.0
+protobuf==4.25.3
+psutil==6.1.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py-cpuinfo==9.0.0
+pyOpenSSL==24.2.1
+pyairports==2.1.1
+pyarrow-hotfix==0.6
+pyarrow==18.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pycountry==24.6.1
+pycparser==2.22
+pycryptodomex==3.21.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+pyparsing==3.2.0
+python-dateutil==2.9.0
+python-dotenv==1.0.1
+python-snappy==0.7.3
+pytz==2024.2
+pyzmq==26.2.0
+quantile-python==1.1
+questionary==2.0.1
+ray==2.39.0
+referencing==0.35.1
+regex==2023.12.25
+requests==2.32.3
+rich==13.9.4
+rotary-emb==0.5.2
+rpds-py==0.21.0
+rsa==4.9
+ruamel.yaml.clib==0.2.12
+ruamel.yaml==0.18.6
+s3transfer==0.10.3
+safetensors==0.4.5
+sentencepiece==0.1.99
+sentry-sdk==2.18.0
+setproctitle==1.3.4
+setuptools==75.6.0
+shapely==2.0.6
+simple-parsing==0.1.6
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+stack-data==0.6.3
+starlette==0.41.3
+sympy==1.13.1
+tiktoken==0.7.0
+tokenizers==0.19.1
+tomli==2.0.1
+torch==2.4.1
+torchvision==0.19.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.44.2
+triton==3.0.0
+typeguard==4.3.0
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+typing_extensions==4.12.2
+tzdata==2024.2
+urllib3==2.2.3
+uvicorn==0.32.0
+uvloop==0.21.0
+validators==0.34.0
+vertexai==1.71.1
+wandb==0.17.3
+watchfiles==0.24.0
+wcwidth==0.2.13
+websockets==11.0.3
+wheel==0.43.0
+wheel==0.45.1
+xformers==0.0.28.post1
+xxhash==3.5.0
+yarl==1.17.2
+zipp==3.19.2
+zipp==3.21.0
+zstandard==0.23.0
+zstd==1.5.5.1

wandb/run-20250102_021927-pw8rud5e/files/wandb-metadata.json

@@ -0,0 +1,705 @@
+{
+    "os": "Linux-6.8.0-48-generic-x86_64-with-glibc2.39",
+    "python": "3.10.0",
+    "heartbeatAt": "2025-01-02T02:19:28.293281",
+    "startedAt": "2025-01-02T02:19:27.822029",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--model_family",
+        "llama",
+        "--apply_instruct_masks",
+        "--token_scaled_loss",
+        "--seq_parallel_size",
+        "8",
+        "--report_to",
+        "wandb",
+        "--do_train",
+        "--model_name_or_path",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--config_name",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--tokenizer_name",
+        "/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/",
+        "--run_name",
+        "_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_",
+        "--output_dir",
+        "/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_",
+        "--config_overrides_json",
+        "",
+        "--gradient_accumulation_steps",
+        "2",
+        "--per_device_train_batch_size",
+        "1",
+        "--bf16",
+        "--learning_rate",
+        "1e-6",
+        "--min_lr_ratio",
+        "0.1",
+        "--lr_scheduler_type",
+        "cosine",
+        "--max_grad_norm",
+        "1.0",
+        "--adam_beta1",
+        "0.9",
+        "--adam_beta2",
+        "0.95",
+        "--weight_decay",
+        "0.1",
+        "--warmup_ratio",
+        "0.05",
+        "--optim",
+        "adamw_torch",
+        "--logging_steps",
+        "1",
+        "--log_level",
+        "info",
+        "--save_steps",
+        "200",
+        "--dataloader_num_workers",
+        "1",
+        "--disable_tqdm",
+        "true",
+        "--use_fast_tokenizer",
+        "false",
+        "--remove_unused_columns",
+        "false",
+        "--ddp_find_unused_parameters",
+        "false",
+        "--fsdp",
+        "auto_wrap offload",
+        "--gradient_checkpointing",
+        "--tokenized_mds_train",
+        "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/data/ft/bookclaim_balanced_pack_complete",
+        "--cuda_empty_cache",
+        "--num_train_epochs",
+        "1"
+    ],
+    "state": "running",
+    "program": "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py",
+    "codePathLocal": "finetune.py",
+    "codePath": "prolong-final/finetune.py",
+    "git": {
+        "remote": "https://github.com/chtmp223/BookGen-dev.git",
+        "commit": "0e796521430a0f767be7c4dadba5c2fcaee1f909"
+    },
+    "email": "[email protected]",
+    "root": "/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev",
+    "host": "gpu020",
+    "username": "ctpham_umass_edu",
+    "executable": "/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/bin/python3.10",
+    "cpu_count": 112,
+    "cpu_count_logical": 112,
+    "cpu_freq": {
+        "current": 958.6112589285714,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 784.548,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 761.056,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.579,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.219,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2400.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1500.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2400.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2000.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 786.524,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1994.764,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.651,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1717.12,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 1300.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 2005.181,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 783.387,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 782.915,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 784.371,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 438.487850189209,
+            "used": 18.224109649658203
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-80GB",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        }
+    ],
+    "memory": {
+        "total": 2015.3287239074707
+    }
+}

wandb/run-20250102_021927-pw8rud5e/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss": 0.2278, "train/grad_norm": 8.256059646606445, "train/learning_rate": 8.018078351076653e-07, "train/epoch": 0.07431516137781394, "train/num_input_tokens_seen": 2671771648, "train/global_step": 1274, "_timestamp": 1735793002.997822, "_runtime": 8635.158229112625, "_step": 273}

wandb/run-20250102_021927-pw8rud5e/logs/debug.log

@@ -0,0 +1,31 @@
+2025-01-02 02:19:27,831 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Current SDK version is 0.17.3
+2025-01-02 02:19:27,831 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Configure stats pid to 2085425
+2025-01-02 02:19:27,831 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Loading settings from /home/ctpham_umass_edu/.config/wandb/settings
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Loading settings from /work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/wandb/settings
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'project': 'prolong', 'api_key': '***REDACTED***', 'mode': 'online'}
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'prolong-final/finetune.py', 'program_abspath': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py', 'program': '/work/pi_miyyer_umass_edu/ctpham/BookClaim-dev/prolong-final/finetune.py'}
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_init.py:_log_setup():520] Logging user logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/logs/debug.log
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_init.py:_log_setup():521] Logging internal logs to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/wandb/run-20250102_021927-pw8rud5e/logs/debug-internal.log
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_init.py:init():560] calling init triggers
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_init.py:init():567] wandb.init called with sweep_config: {}
+config: {}
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_init.py:init():610] starting backend
+2025-01-02 02:19:27,832 INFO    MainThread:2085425 [wandb_init.py:init():614] setting up manager
+2025-01-02 02:19:27,835 INFO    MainThread:2085425 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-02 02:19:27,839 INFO    MainThread:2085425 [wandb_init.py:init():622] backend started and connected
+2025-01-02 02:19:27,847 INFO    MainThread:2085425 [wandb_init.py:init():711] updated telemetry
+2025-01-02 02:19:27,879 INFO    MainThread:2085425 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout
+2025-01-02 02:19:28,136 INFO    MainThread:2085425 [wandb_run.py:_on_init():2402] communicating current version
+2025-01-02 02:19:28,196 INFO    MainThread:2085425 [wandb_run.py:_on_init():2411] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-02 02:19:28,196 INFO    MainThread:2085425 [wandb_init.py:init():795] starting run threads in backend
+2025-01-02 02:19:35,037 INFO    MainThread:2085425 [wandb_run.py:_console_start():2380] atexit reg
+2025-01-02 02:19:35,037 INFO    MainThread:2085425 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2025-01-02 02:19:35,037 INFO    MainThread:2085425 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2025-01-02 02:19:35,037 INFO    MainThread:2085425 [wandb_run.py:_redirect():2325] Redirects installed.
+2025-01-02 02:19:35,039 INFO    MainThread:2085425 [wandb_init.py:init():838] run started, returning control to user process
+2025-01-02 02:19:35,041 INFO    MainThread:2085425 [wandb_run.py:_config_callback():1382] config_cb None None {'vocab_size': 128256, 'max_position_embeddings': 131072, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 32, 'num_attention_heads': 32, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 500000.0, 'rope_scaling': {'factor': 8.0, 'low_freq_factor': 1.0, 'high_freq_factor': 4.0, 'original_max_position_embeddings': 8192, 'rope_type': 'llama3'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 128000, 'pad_token_id': 0, 'eos_token_id': [128001, 128008, 128009], 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '/datasets/ai/llama3/meta-llama/models--meta-llama--Meta-Llama-3.1-8B-Instruct/snapshots/5206a32e0bd3067aef1ce90f5528ade7d866253f/', 'transformers_version': '4.44.2', 'model_type': 'llama', 'output_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'overwrite_output_dir': False, 'do_train': True, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-06, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.05, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/runs/Jan02_02-12-22_gpu020', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1.0, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 200, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 1, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_', 'disable_tqdm': True, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': True, 'fsdp': ['auto_wrap', 'offload'], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'eval_use_gather_object': False, 'min_lr_ratio': 0.1, 'cuda_empty_cache': True, 'streaming_dataset': True, 'seq_parallel_size': 8}
+2025-01-02 02:19:35,044 INFO    MainThread:2085425 [wandb_config.py:__setitem__():151] config set model/num_parameters = 1003782656 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x78e4f41c3100>>
+2025-01-02 02:19:35,044 INFO    MainThread:2085425 [wandb_run.py:_config_callback():1382] config_cb model/num_parameters 1003782656 None

wandb/run-20250102_021927-pw8rud5e/run-pw8rud5e.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b6cc9012c5f1b9992a864e01d9766956c97b137b6869526158e3838a3707f24
+size 1737525

wandb/run-20250102_074844-1ecgrehs/files/conda-environment.yaml

@@ -0,0 +1,233 @@
+name: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - bzip2=1.0.8=h4bc722e_7
+  - ca-certificates=2024.12.14=hbcca054_0
+  - ld_impl_linux-64=2.43=h712a8e2_2
+  - libffi=3.4.2=h7f98852_5
+  - libgcc=14.2.0=h77fa898_1
+  - libgcc-ng=14.2.0=h69a702a_1
+  - libgomp=14.2.0=h77fa898_1
+  - liblzma=5.6.3=hb9d3cd8_1
+  - liblzma-devel=5.6.3=hb9d3cd8_1
+  - libnsl=2.0.1=hd590300_0
+  - libsqlite=3.47.2=hee588c1_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.3.1=hb9d3cd8_2
+  - ncurses=6.5=he02047a_1
+  - openssl=3.4.0=hb9d3cd8_0
+  - pip=24.3.1=pyh8b19718_2
+  - python=3.10.0=h543edf9_3_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=75.6.0=pyhff2d567_1
+  - sqlite=3.47.2=h9eae976_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - wheel=0.45.1=pyhd8ed1ab_1
+  - xz=5.6.3=hbcc6ac9_1
+  - xz-gpl-tools=5.6.3=hbcc6ac9_1
+  - xz-tools=5.6.3=hb9d3cd8_1
+  - pip:
+      - accelerate==0.32.1
+      - aiohappyeyeballs==2.4.3
+      - aiohttp==3.11.2
+      - aioprometheus==23.12.0
+      - aiosignal==1.3.1
+      - annotated-types==0.7.0
+      - anthropic==0.39.0
+      - anyio==4.6.2.post1
+      - argcomplete==3.5.1
+      - arrow==1.3.0
+      - async-timeout==5.0.1
+      - attrs==24.2.0
+      - azure-core==1.32.0
+      - azure-identity==1.19.0
+      - azure-storage-blob==12.24.0
+      - azure-storage-file-datalake==12.18.0
+      - backoff==2.2.1
+      - bcrypt==4.2.0
+      - blobfile==3.0.0
+      - boto3==1.35.63
+      - botocore==1.35.63
+      - brotli==1.1.0
+      - cachetools==5.5.0
+      - certifi==2024.8.30
+      - cffi==1.17.1
+      - charset-normalizer==3.4.0
+      - circuitbreaker==2.0.0
+      - click==8.1.7
+      - cloudpickle==3.1.0
+      - compressed-tensors==0.8.0
+      - contourpy==1.3.1
+      - cramjam==2.9.0
+      - cryptography==43.0.3
+      - cycler==0.12.1
+      - datasets==2.20.0
+      - debugpy==1.8.11
+      - dill==0.3.8
+      - diskcache==5.6.3
+      - distro==1.9.0
+      - docker-pycreds==0.4.0
+      - docstring-parser==0.16
+      - einops==0.8.0
+      - fastapi==0.115.5
+      - filelock==3.16.1
+      - flash-attn==2.6.1
+      - fonttools==4.55.0
+      - frozenlist==1.5.0
+      - fsspec==2024.5.0
+      - gguf==0.10.0
+      - gitdb==4.0.11
+      - gitpython==3.1.43
+      - google-api-core==2.23.0
+      - google-auth==2.36.0
+      - google-cloud-aiplatform==1.71.1
+      - google-cloud-bigquery==3.27.0
+      - google-cloud-core==2.4.1
+      - google-cloud-resource-manager==1.13.1
+      - google-cloud-storage==2.10.0
+      - google-crc32c==1.6.0
+      - google-resumable-media==2.7.2
+      - googleapis-common-protos==1.66.0
+      - gql==3.5.0
+      - graphql-core==3.2.5
+      - grpc-google-iam-v1==0.13.1
+      - grpcio==1.68.0
+      - grpcio-status==1.62.3
+      - h11==0.14.0
+      - httpcore==1.0.7
+      - httptools==0.6.4
+      - httpx==0.27.2
+      - huggingface-hub==0.26.2
+      - idna==3.10
+      - importlib-metadata==8.5.0
+      - interegular==0.3.3
+      - ipython==8.18.0
+      - isodate==0.7.2
+      - jedi==0.19.2
+      - jinja2==3.1.4
+      - jiter==0.7.1
+      - jmespath==1.0.1
+      - jsonschema==4.23.0
+      - jsonschema-specifications==2024.10.1
+      - kiwisolver==1.4.7
+      - lark==1.2.2
+      - llvmlite==0.43.0
+      - lm-format-enforcer==0.10.9
+      - lxml==5.3.0
+      - markdown-it-py==3.0.0
+      - markupsafe==3.0.2
+      - matplotlib==3.9.2
+      - mdurl==0.1.2
+      - mosaicml-cli==0.5.34
+      - mosaicml-streaming==0.8.1
+      - mpmath==1.3.0
+      - msal==1.31.1
+      - msal-extensions==1.2.0
+      - msgpack==1.1.0
+      - msgspec==0.18.6
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - networkx==3.4.2
+      - ninja==1.11.1.1
+      - numba==0.60.0
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==9.1.0.70
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-ml-py==12.560.30
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.4.127
+      - nvidia-nvtx-cu12==12.1.105
+      - oci==2.138.1
+      - openai==1.54.5
+      - opencv-python-headless==4.10.0.84
+      - orjson==3.10.11
+      - outlines==0.0.46
+      - packaging==24.1
+      - pandas==2.2.1
+      - paramiko==3.5.0
+      - partial-json-parser==0.2.1.1.post4
+      - pillow==10.4.0
+      - portalocker==2.10.1
+      - prometheus-client==0.21.0
+      - prometheus-fastapi-instrumentator==7.0.0
+      - prompt-toolkit==3.0.36
+      - propcache==0.2.0
+      - proto-plus==1.25.0
+      - protobuf==4.25.3
+      - py-cpuinfo==9.0.0
+      - pyairports==2.1.1
+      - pyarrow==18.0.0
+      - pyarrow-hotfix==0.6
+      - pyasn1==0.6.1
+      - pyasn1-modules==0.4.1
+      - pycountry==24.6.1
+      - pycparser==2.22
+      - pycryptodomex==3.21.0
+      - pydantic==2.9.2
+      - pydantic-core==2.23.4
+      - pyjwt==2.10.0
+      - pynacl==1.5.0
+      - pyopenssl==24.2.1
+      - pyparsing==3.2.0
+      - python-dateutil==2.9.0
+      - python-dotenv==1.0.1
+      - python-snappy==0.7.3
+      - pytz==2024.2
+      - pyyaml==6.0.2
+      - quantile-python==1.1
+      - questionary==2.0.1
+      - ray==2.39.0
+      - referencing==0.35.1
+      - regex==2023.12.25
+      - requests==2.32.3
+      - rich==13.9.4
+      - rotary-emb==0.5.2
+      - rpds-py==0.21.0
+      - rsa==4.9
+      - ruamel-yaml==0.18.6
+      - ruamel-yaml-clib==0.2.12
+      - s3transfer==0.10.3
+      - safetensors==0.4.5
+      - sentencepiece==0.1.99
+      - sentry-sdk==2.18.0
+      - setproctitle==1.3.4
+      - shapely==2.0.6
+      - simple-parsing==0.1.6
+      - smmap==5.0.1
+      - sniffio==1.3.1
+      - starlette==0.41.3
+      - sympy==1.13.1
+      - tiktoken==0.7.0
+      - tokenizers==0.19.1
+      - torch==2.4.1
+      - torchvision==0.19.1
+      - tqdm==4.66.4
+      - transformers==4.44.2
+      - triton==3.0.0
+      - types-python-dateutil==2.9.0.20241003
+      - tzdata==2024.2
+      - urllib3==2.2.3
+      - uvicorn==0.32.0
+      - uvloop==0.21.0
+      - validators==0.34.0
+      - vertexai==1.71.1
+      - wandb==0.17.3
+      - watchfiles==0.24.0
+      - websockets==11.0.3
+      - xformers==0.0.28.post1
+      - xxhash==3.5.0
+      - yarl==1.17.2
+      - zipp==3.21.0
+      - zstandard==0.23.0
+      - zstd==1.5.5.1
+prefix: /scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final