Model save

README.md

@@ -20,7 +20,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0244
+- Loss: 1.0546
 
 ## Model description
 
@@ -40,12 +40,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 4e-05
-- train_batch_size: 3
-- eval_batch_size: 3
+- train_batch_size: 6
+- eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- gradient_accumulation_steps: 40
-- total_train_batch_size: 120
+- gradient_accumulation_steps: 36
+- total_train_batch_size: 216
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - num_epochs: 1
@@ -54,8 +54,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 0.9699        | 0.38  | 80   | 1.0432          |
-| 0.9576        | 0.77  | 160  | 1.0250          |
+| 1.1912        | 0.28  | 10   | 1.1099          |
+| 1.1238        | 0.55  | 20   | 1.0655          |
+| 1.1258        | 0.83  | 30   | 1.0550          |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57de9f65961ee02019b9e09abe2f8108d672f1d339c0c27346ccf40f27ed3e4e
+oid sha256:36f835eb9195a1de5a20430109f1d198bee36ccd393dc5dbe881b49aea860788
 size 26362152

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 1.0,
-    "eval_loss": 1.0243600606918335,
-    "eval_runtime": 26.1874,
-    "eval_samples": 783,
-    "eval_samples_per_second": 4.773,
-    "eval_steps_per_second": 1.604,
-    "train_loss": 0.5025384334856243,
-    "train_runtime": 8965.5847,
-    "train_samples": 115131,
-    "train_samples_per_second": 2.798,
-    "train_steps_per_second": 0.023
+    "eval_loss": 1.0545984506607056,
+    "eval_runtime": 9.2861,
+    "eval_samples": 130,
+    "eval_samples_per_second": 5.061,
+    "eval_steps_per_second": 1.292,
+    "train_loss": 1.1544433269235823,
+    "train_runtime": 5078.5964,
+    "train_samples": 29726,
+    "train_samples_per_second": 1.537,
+    "train_steps_per_second": 0.007
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_loss": 1.0243600606918335,
-    "eval_runtime": 26.1874,
-    "eval_samples": 783,
-    "eval_samples_per_second": 4.773,
-    "eval_steps_per_second": 1.604
+    "eval_loss": 1.0545984506607056,
+    "eval_runtime": 9.2861,
+    "eval_samples": 130,
+    "eval_samples_per_second": 5.061,
+    "eval_steps_per_second": 1.292
 }

runs/Jan13_19-31-59_98f107f1aa39/events.out.tfevents.1705174497.98f107f1aa39.154347.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd69b424d6c6d97ea528767f2e0f4baedd3db13474839bd4f0a724058db48e10
-size 6159
+oid sha256:19b23ffb1193103c8874ec5d6b80f396589fdaf1ea2fab93a29bb79b481a657f
+size 7235

runs/Jan13_19-31-59_98f107f1aa39/events.out.tfevents.1705179585.98f107f1aa39.154347.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:771da7e33030b7c506cff2ebedb68fd5813ed7af9fee67bbca483a22096bc161
+size 354

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5025384334856243,
-    "train_runtime": 8965.5847,
-    "train_samples": 115131,
-    "train_samples_per_second": 2.798,
-    "train_steps_per_second": 0.023
+    "train_loss": 1.1544433269235823,
+    "train_runtime": 5078.5964,
+    "train_samples": 29726,
+    "train_samples_per_second": 1.537,
+    "train_steps_per_second": 0.007
 }

trainer_state.json

@@ -1,298 +1,102 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9996412770536889,
-  "eval_steps": 80,
-  "global_step": 209,
+  "epoch": 0.9961568024596464,
+  "eval_steps": 10,
+  "global_step": 36,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "learning_rate": 3.9997740569453936e-05,
-      "loss": 1.3377,
+      "epoch": 0.03,
+      "learning_rate": 3.9923893961834914e-05,
+      "loss": 1.3268,
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 3.9943539757443494e-05,
-      "loss": 1.1879,
+      "epoch": 0.14,
+      "learning_rate": 3.812615574073301e-05,
+      "loss": 1.2548,
       "step": 5
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 3.97744778056729e-05,
-      "loss": 1.1238,
+      "epoch": 0.28,
+      "learning_rate": 3.285575219373079e-05,
+      "loss": 1.1912,
       "step": 10
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 3.949376867256863e-05,
-      "loss": 1.0839,
-      "step": 15
-    },
-    {
-      "epoch": 0.1,
-      "learning_rate": 3.9102997248704994e-05,
-      "loss": 1.0787,
-      "step": 20
-    },
-    {
-      "epoch": 0.12,
-      "learning_rate": 3.8604369839019515e-05,
-      "loss": 1.0363,
-      "step": 25
-    },
-    {
-      "epoch": 0.14,
-      "learning_rate": 3.800070170596182e-05,
-      "loss": 1.0079,
-      "step": 30
-    },
-    {
-      "epoch": 0.17,
-      "learning_rate": 3.729540117445352e-05,
-      "loss": 1.0173,
-      "step": 35
-    },
-    {
-      "epoch": 0.19,
-      "learning_rate": 3.6492450388403034e-05,
-      "loss": 1.0162,
-      "step": 40
-    },
-    {
-      "epoch": 0.22,
-      "learning_rate": 3.559638282742449e-05,
-      "loss": 1.0247,
-      "step": 45
-    },
-    {
-      "epoch": 0.24,
-      "learning_rate": 3.461225771070188e-05,
-      "loss": 1.0066,
-      "step": 50
-    },
-    {
-      "epoch": 0.26,
-      "learning_rate": 3.354563143251483e-05,
-      "loss": 0.9969,
-      "step": 55
-    },
-    {
-      "epoch": 0.29,
-      "learning_rate": 3.2402526190701667e-05,
-      "loss": 1.0205,
-      "step": 60
-    },
-    {
-      "epoch": 0.31,
-      "learning_rate": 3.1189395985184464e-05,
-      "loss": 0.9882,
-      "step": 65
-    },
-    {
-      "epoch": 0.33,
-      "learning_rate": 2.9913090178528815e-05,
-      "loss": 0.9749,
-      "step": 70
-    },
-    {
-      "epoch": 0.36,
-      "learning_rate": 2.858081482427673e-05,
-      "loss": 0.9849,
-      "step": 75
-    },
-    {
-      "epoch": 0.38,
-      "learning_rate": 2.7200091981393524e-05,
-      "loss": 0.9699,
-      "step": 80
-    },
-    {
-      "epoch": 0.38,
-      "eval_loss": 1.043211817741394,
-      "eval_runtime": 26.2196,
-      "eval_samples_per_second": 4.767,
-      "eval_steps_per_second": 1.602,
-      "step": 80
-    },
-    {
-      "epoch": 0.41,
-      "learning_rate": 2.577871724454045e-05,
-      "loss": 0.9874,
-      "step": 85
-    },
-    {
-      "epoch": 0.43,
-      "learning_rate": 2.4324715729958146e-05,
-      "loss": 0.9723,
-      "step": 90
-    },
-    {
-      "epoch": 0.45,
-      "learning_rate": 2.2846296765465708e-05,
-      "loss": 0.9884,
-      "step": 95
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 2.1351807540396666e-05,
-      "loss": 0.9596,
-      "step": 100
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 1.9849685977165566e-05,
-      "loss": 0.9784,
-      "step": 105
+      "epoch": 0.28,
+      "eval_loss": 1.109934687614441,
+      "eval_runtime": 9.2946,
+      "eval_samples_per_second": 5.057,
+      "eval_steps_per_second": 1.291,
+      "step": 10
     },
     {
-      "epoch": 0.53,
-      "learning_rate": 1.8348413090553356e-05,
-      "loss": 0.9715,
-      "step": 110
+      "epoch": 0.42,
+      "learning_rate": 2.5176380902050418e-05,
+      "loss": 1.1433,
+      "step": 15
     },
     {
       "epoch": 0.55,
-      "learning_rate": 1.6856465103692203e-05,
-      "loss": 0.9627,
-      "step": 115
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 1.5382265591104088e-05,
-      "loss": 0.96,
-      "step": 120
-    },
-    {
-      "epoch": 0.6,
-      "learning_rate": 1.3934137918994753e-05,
-      "loss": 0.9743,
-      "step": 125
-    },
-    {
-      "epoch": 0.62,
-      "learning_rate": 1.2520258251326212e-05,
-      "loss": 0.9661,
-      "step": 130
-    },
-    {
-      "epoch": 0.65,
-      "learning_rate": 1.1148609386996692e-05,
-      "loss": 0.9676,
-      "step": 135
+      "learning_rate": 1.6527036446661396e-05,
+      "loss": 1.1238,
+      "step": 20
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 9.826935688764434e-06,
-      "loss": 0.9842,
-      "step": 140
+      "epoch": 0.55,
+      "eval_loss": 1.065536379814148,
+      "eval_runtime": 9.2887,
+      "eval_samples_per_second": 5.06,
+      "eval_steps_per_second": 1.292,
+      "step": 20
     },
     {
       "epoch": 0.69,
-      "learning_rate": 8.562699358387723e-06,
-      "loss": 0.9628,
-      "step": 145
-    },
-    {
-      "epoch": 0.72,
-      "learning_rate": 7.3630383048527255e-06,
-      "loss": 0.9603,
-      "step": 150
-    },
-    {
-      "epoch": 0.74,
-      "learning_rate": 6.234725843566269e-06,
-      "loss": 0.9626,
-      "step": 155
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 5.184132454052731e-06,
-      "loss": 0.9576,
-      "step": 160
-    },
-    {
-      "epoch": 0.77,
-      "eval_loss": 1.0250179767608643,
-      "eval_runtime": 26.1866,
-      "eval_samples_per_second": 4.773,
-      "eval_steps_per_second": 1.604,
-      "step": 160
-    },
-    {
-      "epoch": 0.79,
-      "learning_rate": 4.217189812072131e-06,
-      "loss": 0.9659,
-      "step": 165
-    },
-    {
-      "epoch": 0.81,
-      "learning_rate": 3.3393572992349156e-06,
-      "loss": 0.9655,
-      "step": 170
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 2.5555911792009624e-06,
-      "loss": 0.9501,
-      "step": 175
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 1.8703166144947427e-06,
-      "loss": 0.9754,
-      "step": 180
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 1.2874026819303698e-06,
-      "loss": 0.9497,
-      "step": 185
-    },
-    {
-      "epoch": 0.91,
-      "learning_rate": 8.101405277100549e-07,
-      "loss": 0.9477,
-      "step": 190
+      "learning_rate": 8.528471272979083e-06,
+      "loss": 1.1102,
+      "step": 25
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 4.412247855328322e-07,
-      "loss": 0.9624,
-      "step": 195
+      "epoch": 0.83,
+      "learning_rate": 2.679491924311226e-06,
+      "loss": 1.1258,
+      "step": 30
     },
     {
-      "epoch": 0.96,
-      "learning_rate": 1.8273836262732824e-07,
-      "loss": 0.959,
-      "step": 200
+      "epoch": 0.83,
+      "eval_loss": 1.055001974105835,
+      "eval_runtime": 9.2909,
+      "eval_samples_per_second": 5.059,
+      "eval_steps_per_second": 1.292,
+      "step": 30
     },
     {
-      "epoch": 0.98,
-      "learning_rate": 3.614067960701961e-08,
-      "loss": 0.9523,
-      "step": 205
+      "epoch": 0.97,
+      "learning_rate": 7.61060381650891e-08,
+      "loss": 1.1272,
+      "step": 35
     },
     {
       "epoch": 1.0,
-      "step": 209,
-      "total_flos": 3.512657938470666e+17,
-      "train_loss": 0.5025384334856243,
-      "train_runtime": 8965.5847,
-      "train_samples_per_second": 2.798,
-      "train_steps_per_second": 0.023
+      "step": 36,
+      "total_flos": 1.0010669722946765e+17,
+      "train_loss": 1.1544433269235823,
+      "train_runtime": 5078.5964,
+      "train_samples_per_second": 1.537,
+      "train_steps_per_second": 0.007
     }
   ],
   "logging_steps": 5,
-  "max_steps": 209,
+  "max_steps": 36,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 50,
-  "total_flos": 3.512657938470666e+17,
-  "train_batch_size": 3,
+  "save_steps": 20,
+  "total_flos": 1.0010669722946765e+17,
+  "train_batch_size": 6,
   "trial_name": null,
   "trial_params": null
 }