End of training

README.md

@@ -2,11 +2,23 @@
 library_name: transformers
 tags:
 - generated_from_trainer
+datasets:
+- kanishka/babylm2-rewritten-clean-spacy
 metrics:
 - accuracy
 model-index:
 - name: opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3
-  results: []
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: kanishka/babylm2-rewritten-clean-spacy
+      type: kanishka/babylm2-rewritten-clean-spacy
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.47868057440510814
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -14,7 +26,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the kanishka/babylm2-rewritten-clean-spacy dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.6840
 - Accuracy: 0.4787

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 20.0,
-    "eval_accuracy": 0.47877642614021604,
-    "eval_loss": 2.6820449829101562,
-    "eval_runtime": 71.9233,
-    "eval_samples": 60701,
-    "eval_samples_per_second": 843.969,
-    "eval_steps_per_second": 13.195,
-    "perplexity": 14.614950080315884,
-    "total_flos": 1.50902942072832e+18,
-    "train_loss": 2.805498681169875,
-    "train_runtime": 30524.4807,
-    "train_samples": 577526,
-    "train_samples_per_second": 378.402,
-    "train_steps_per_second": 1.478
+    "epoch": 19.991464360935595,
+    "eval_accuracy": 0.47868057440510814,
+    "eval_loss": 2.6840312480926514,
+    "eval_runtime": 71.5689,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 847.854,
+    "eval_steps_per_second": 13.26,
+    "perplexity": 14.644008095713023,
+    "total_flos": 1.507910045663232e+18,
+    "train_loss": 2.8050402250099604,
+    "train_runtime": 30336.2728,
+    "train_samples": 577344,
+    "train_samples_per_second": 380.629,
+    "train_steps_per_second": 1.487
 }

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 20.0,
-    "eval_accuracy": 0.47877642614021604,
-    "eval_loss": 2.6820449829101562,
-    "eval_runtime": 71.9233,
-    "eval_samples": 60701,
-    "eval_samples_per_second": 843.969,
-    "eval_steps_per_second": 13.195,
-    "perplexity": 14.614950080315884
+    "epoch": 19.991464360935595,
+    "eval_accuracy": 0.47868057440510814,
+    "eval_loss": 2.6840312480926514,
+    "eval_runtime": 71.5689,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 847.854,
+    "eval_steps_per_second": 13.26,
+    "perplexity": 14.644008095713023
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 20.0,
-    "total_flos": 1.50902942072832e+18,
-    "train_loss": 2.805498681169875,
-    "train_runtime": 30524.4807,
-    "train_samples": 577526,
-    "train_samples_per_second": 378.402,
-    "train_steps_per_second": 1.478
+    "epoch": 19.991464360935595,
+    "total_flos": 1.507910045663232e+18,
+    "train_loss": 2.8050402250099604,
+    "train_runtime": 30336.2728,
+    "train_samples": 577344,
+    "train_samples_per_second": 380.629,
+    "train_steps_per_second": 1.487
 }

trainer_state.json

@@ -1,520 +1,520 @@
 {
-  "best_metric": 2.6820449829101562,
-  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3/checkpoint-45120",
-  "epoch": 20.0,
+  "best_metric": 2.6840312480926514,
+  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-42_1e-3/checkpoint-45100",
+  "epoch": 19.991464360935595,
   "eval_steps": 500,
-  "global_step": 45120,
+  "global_step": 45100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.4432624113475177,
-      "grad_norm": 0.469483345746994,
+      "epoch": 0.4434098215275468,
+      "grad_norm": 0.4883245825767517,
       "learning_rate": 3.125e-05,
-      "loss": 5.5946,
+      "loss": 5.5896,
       "step": 1000
     },
     {
-      "epoch": 0.8865248226950354,
-      "grad_norm": 0.6834925413131714,
+      "epoch": 0.8868196430550936,
+      "grad_norm": 0.6184232831001282,
       "learning_rate": 6.25e-05,
-      "loss": 4.101,
+      "loss": 4.1044,
       "step": 2000
     },
     {
       "epoch": 1.0,
-      "eval_accuracy": 0.36108013855119486,
-      "eval_loss": 3.813789129257202,
-      "eval_runtime": 74.5004,
-      "eval_samples_per_second": 814.774,
-      "eval_steps_per_second": 12.738,
+      "eval_accuracy": 0.3604034665942844,
+      "eval_loss": 3.820427417755127,
+      "eval_runtime": 74.0095,
+      "eval_samples_per_second": 819.894,
+      "eval_steps_per_second": 12.823,
       "step": 2256
     },
     {
-      "epoch": 1.3297872340425532,
-      "grad_norm": 0.5719049572944641,
+      "epoch": 1.3298969072164948,
+      "grad_norm": 0.5272237658500671,
       "learning_rate": 9.375e-05,
-      "loss": 3.7004,
+      "loss": 3.6981,
       "step": 3000
     },
     {
-      "epoch": 1.773049645390071,
-      "grad_norm": 0.5357337594032288,
+      "epoch": 1.7733067287440418,
+      "grad_norm": 0.580745279788971,
       "learning_rate": 0.000125,
-      "loss": 3.445,
+      "loss": 3.4457,
       "step": 4000
     },
     {
       "epoch": 2.0,
-      "eval_accuracy": 0.4097865752122829,
-      "eval_loss": 3.29646635055542,
-      "eval_runtime": 74.2471,
-      "eval_samples_per_second": 817.554,
-      "eval_steps_per_second": 12.782,
+      "eval_accuracy": 0.4093216099887549,
+      "eval_loss": 3.304572105407715,
+      "eval_runtime": 74.3697,
+      "eval_samples_per_second": 815.923,
+      "eval_steps_per_second": 12.761,
       "step": 4512
     },
     {
-      "epoch": 2.2163120567375887,
-      "grad_norm": 0.5515570640563965,
+      "epoch": 2.2163839929054427,
+      "grad_norm": 0.5752166509628296,
       "learning_rate": 0.00015625,
-      "loss": 3.2498,
+      "loss": 3.2482,
       "step": 5000
     },
     {
-      "epoch": 2.6595744680851063,
-      "grad_norm": 0.5060694813728333,
+      "epoch": 2.6597938144329896,
+      "grad_norm": 0.45855629444122314,
       "learning_rate": 0.0001875,
-      "loss": 3.1279,
+      "loss": 3.13,
       "step": 6000
     },
     {
       "epoch": 3.0,
-      "eval_accuracy": 0.43082928827286177,
-      "eval_loss": 3.0860400199890137,
-      "eval_runtime": 74.3001,
-      "eval_samples_per_second": 816.97,
-      "eval_steps_per_second": 12.773,
+      "eval_accuracy": 0.42987378339602156,
+      "eval_loss": 3.0944786071777344,
+      "eval_runtime": 73.3184,
+      "eval_samples_per_second": 827.624,
+      "eval_steps_per_second": 12.944,
       "step": 6768
     },
     {
-      "epoch": 3.102836879432624,
-      "grad_norm": 0.4440550208091736,
+      "epoch": 3.102871078594391,
+      "grad_norm": 0.4158306419849396,
       "learning_rate": 0.00021875,
-      "loss": 3.0384,
+      "loss": 3.0338,
       "step": 7000
     },
     {
-      "epoch": 3.546099290780142,
-      "grad_norm": 0.4073255956172943,
+      "epoch": 3.5462809001219378,
+      "grad_norm": 0.3917515277862549,
       "learning_rate": 0.00025,
-      "loss": 2.9651,
+      "loss": 2.9667,
       "step": 8000
     },
     {
-      "epoch": 3.9893617021276597,
-      "grad_norm": 0.3835934102535248,
+      "epoch": 3.9896907216494846,
+      "grad_norm": 0.3958011865615845,
       "learning_rate": 0.00028125000000000003,
-      "loss": 2.9218,
+      "loss": 2.9219,
       "step": 9000
     },
     {
       "epoch": 4.0,
-      "eval_accuracy": 0.4411697193992669,
-      "eval_loss": 2.9823455810546875,
-      "eval_runtime": 73.4266,
-      "eval_samples_per_second": 826.69,
-      "eval_steps_per_second": 12.924,
+      "eval_accuracy": 0.4403801362337948,
+      "eval_loss": 2.988952398300171,
+      "eval_runtime": 72.5536,
+      "eval_samples_per_second": 836.347,
+      "eval_steps_per_second": 13.08,
       "step": 9024
     },
     {
-      "epoch": 4.432624113475177,
-      "grad_norm": 0.35862067341804504,
+      "epoch": 4.4327679858108855,
+      "grad_norm": 0.3376877009868622,
       "learning_rate": 0.0003125,
-      "loss": 2.8625,
+      "loss": 2.8585,
       "step": 10000
     },
     {
-      "epoch": 4.875886524822695,
-      "grad_norm": 0.33855435252189636,
+      "epoch": 4.876177807338433,
+      "grad_norm": 0.32727962732315063,
       "learning_rate": 0.00034375,
-      "loss": 2.8441,
+      "loss": 2.8444,
       "step": 11000
     },
     {
       "epoch": 5.0,
-      "eval_accuracy": 0.44725845198790215,
-      "eval_loss": 2.9202077388763428,
-      "eval_runtime": 73.099,
-      "eval_samples_per_second": 830.394,
-      "eval_steps_per_second": 12.982,
+      "eval_accuracy": 0.44664624452285856,
+      "eval_loss": 2.928157091140747,
+      "eval_runtime": 72.5793,
+      "eval_samples_per_second": 836.051,
+      "eval_steps_per_second": 13.075,
       "step": 11280
     },
     {
-      "epoch": 5.319148936170213,
-      "grad_norm": 0.3236384987831116,
+      "epoch": 5.319255071499834,
+      "grad_norm": 0.32956644892692566,
       "learning_rate": 0.000375,
-      "loss": 2.7995,
+      "loss": 2.7978,
       "step": 12000
     },
     {
-      "epoch": 5.76241134751773,
-      "grad_norm": 0.3051661252975464,
+      "epoch": 5.762664893027381,
+      "grad_norm": 0.3080673813819885,
       "learning_rate": 0.00040625000000000004,
-      "loss": 2.7865,
+      "loss": 2.7883,
       "step": 13000
     },
     {
       "epoch": 6.0,
-      "eval_accuracy": 0.4512637482794966,
-      "eval_loss": 2.8827643394470215,
-      "eval_runtime": 72.7776,
-      "eval_samples_per_second": 834.061,
-      "eval_steps_per_second": 13.04,
+      "eval_accuracy": 0.4507549730505254,
+      "eval_loss": 2.8910350799560547,
+      "eval_runtime": 72.5685,
+      "eval_samples_per_second": 836.175,
+      "eval_steps_per_second": 13.077,
       "step": 13536
     },
     {
-      "epoch": 6.205673758865248,
-      "grad_norm": 0.29630789160728455,
+      "epoch": 6.205742157188782,
+      "grad_norm": 0.2959093153476715,
       "learning_rate": 0.0004375,
-      "loss": 2.759,
+      "loss": 2.7566,
       "step": 14000
     },
     {
-      "epoch": 6.648936170212766,
-      "grad_norm": 0.27569055557250977,
+      "epoch": 6.649151978716328,
+      "grad_norm": 0.29388415813446045,
       "learning_rate": 0.00046871875,
-      "loss": 2.7432,
+      "loss": 2.7434,
       "step": 15000
     },
     {
       "epoch": 7.0,
-      "eval_accuracy": 0.4541487994350967,
-      "eval_loss": 2.8589529991149902,
-      "eval_runtime": 72.8865,
-      "eval_samples_per_second": 832.815,
-      "eval_steps_per_second": 13.02,
+      "eval_accuracy": 0.4544950043300115,
+      "eval_loss": 2.8579459190368652,
+      "eval_runtime": 72.4898,
+      "eval_samples_per_second": 837.083,
+      "eval_steps_per_second": 13.091,
       "step": 15792
     },
     {
-      "epoch": 7.092198581560283,
-      "grad_norm": 0.2677430808544159,
+      "epoch": 7.09222924287773,
+      "grad_norm": 0.27015742659568787,
       "learning_rate": 0.00049996875,
-      "loss": 2.7319,
+      "loss": 2.7294,
       "step": 16000
     },
     {
-      "epoch": 7.535460992907802,
-      "grad_norm": 0.2510625123977661,
-      "learning_rate": 0.0005311875000000001,
-      "loss": 2.7061,
+      "epoch": 7.535639064405276,
+      "grad_norm": 0.2585032880306244,
+      "learning_rate": 0.00053121875,
+      "loss": 2.7057,
       "step": 17000
     },
     {
-      "epoch": 7.9787234042553195,
-      "grad_norm": 0.23760418593883514,
-      "learning_rate": 0.0005624375,
-      "loss": 2.7146,
+      "epoch": 7.979048885932824,
+      "grad_norm": 0.26894038915634155,
+      "learning_rate": 0.0005624687499999999,
+      "loss": 2.7158,
       "step": 18000
     },
     {
       "epoch": 8.0,
-      "eval_accuracy": 0.4565537086154539,
-      "eval_loss": 2.8377726078033447,
-      "eval_runtime": 72.7887,
-      "eval_samples_per_second": 833.934,
-      "eval_steps_per_second": 13.038,
+      "eval_accuracy": 0.4559880181472721,
+      "eval_loss": 2.842834949493408,
+      "eval_runtime": 72.6498,
+      "eval_samples_per_second": 835.24,
+      "eval_steps_per_second": 13.063,
       "step": 18048
     },
     {
-      "epoch": 8.421985815602836,
-      "grad_norm": 0.23823712766170502,
-      "learning_rate": 0.00059365625,
-      "loss": 2.675,
+      "epoch": 8.422126150094225,
+      "grad_norm": 0.24038437008857727,
+      "learning_rate": 0.0005936875,
+      "loss": 2.6733,
       "step": 19000
     },
     {
-      "epoch": 8.865248226950355,
-      "grad_norm": 0.23026619851589203,
-      "learning_rate": 0.00062490625,
-      "loss": 2.6906,
+      "epoch": 8.865535971621771,
+      "grad_norm": 0.22421102225780487,
+      "learning_rate": 0.0006249375000000001,
+      "loss": 2.6905,
       "step": 20000
     },
     {
       "epoch": 9.0,
-      "eval_accuracy": 0.45826760614791046,
-      "eval_loss": 2.8225581645965576,
-      "eval_runtime": 72.5363,
-      "eval_samples_per_second": 836.836,
-      "eval_steps_per_second": 13.083,
+      "eval_accuracy": 0.4572794602349839,
+      "eval_loss": 2.8298442363739014,
+      "eval_runtime": 72.6455,
+      "eval_samples_per_second": 835.29,
+      "eval_steps_per_second": 13.063,
       "step": 20304
     },
     {
-      "epoch": 9.308510638297872,
-      "grad_norm": 0.22488652169704437,
-      "learning_rate": 0.000656125,
-      "loss": 2.6598,
+      "epoch": 9.308613235783172,
+      "grad_norm": 0.22955693304538727,
+      "learning_rate": 0.0006561562500000001,
+      "loss": 2.6582,
       "step": 21000
     },
     {
-      "epoch": 9.75177304964539,
-      "grad_norm": 0.20617271959781647,
-      "learning_rate": 0.0006873749999999999,
-      "loss": 2.6681,
+      "epoch": 9.75202305731072,
+      "grad_norm": 0.20538607239723206,
+      "learning_rate": 0.00068740625,
+      "loss": 2.6697,
       "step": 22000
     },
     {
       "epoch": 10.0,
-      "eval_accuracy": 0.459518546549771,
-      "eval_loss": 2.813441753387451,
-      "eval_runtime": 72.4442,
-      "eval_samples_per_second": 837.9,
-      "eval_steps_per_second": 13.1,
+      "eval_accuracy": 0.45919856010960747,
+      "eval_loss": 2.816859006881714,
+      "eval_runtime": 72.615,
+      "eval_samples_per_second": 835.64,
+      "eval_steps_per_second": 13.069,
       "step": 22560
     },
     {
-      "epoch": 10.195035460992909,
-      "grad_norm": 0.20597966015338898,
-      "learning_rate": 0.000718625,
-      "loss": 2.6559,
+      "epoch": 10.19510032147212,
+      "grad_norm": 0.2177572250366211,
+      "learning_rate": 0.00071865625,
+      "loss": 2.6506,
       "step": 23000
     },
     {
-      "epoch": 10.638297872340425,
-      "grad_norm": 0.21323370933532715,
-      "learning_rate": 0.0007498437500000001,
-      "loss": 2.6498,
+      "epoch": 10.638510142999667,
+      "grad_norm": 0.2029583603143692,
+      "learning_rate": 0.000749875,
+      "loss": 2.6509,
       "step": 24000
     },
     {
       "epoch": 11.0,
-      "eval_accuracy": 0.4604723054276652,
-      "eval_loss": 2.8047826290130615,
-      "eval_runtime": 72.8612,
-      "eval_samples_per_second": 833.105,
-      "eval_steps_per_second": 13.025,
+      "eval_accuracy": 0.46011432522910284,
+      "eval_loss": 2.807971477508545,
+      "eval_runtime": 72.7484,
+      "eval_samples_per_second": 834.107,
+      "eval_steps_per_second": 13.045,
       "step": 24816
     },
     {
-      "epoch": 11.081560283687944,
-      "grad_norm": 0.21533997356891632,
-      "learning_rate": 0.00078109375,
-      "loss": 2.6487,
+      "epoch": 11.081587407161068,
+      "grad_norm": 0.2110850214958191,
+      "learning_rate": 0.000781125,
+      "loss": 2.6497,
       "step": 25000
     },
     {
-      "epoch": 11.52482269503546,
-      "grad_norm": 0.21542951464653015,
-      "learning_rate": 0.0008123125,
-      "loss": 2.6323,
+      "epoch": 11.524997228688616,
+      "grad_norm": 0.19248805940151215,
+      "learning_rate": 0.000812375,
+      "loss": 2.6322,
       "step": 26000
     },
     {
-      "epoch": 11.96808510638298,
-      "grad_norm": 0.19053979218006134,
-      "learning_rate": 0.0008435625,
-      "loss": 2.6497,
+      "epoch": 11.968407050216163,
+      "grad_norm": 0.18789444863796234,
+      "learning_rate": 0.00084359375,
+      "loss": 2.6494,
       "step": 27000
     },
     {
       "epoch": 12.0,
-      "eval_accuracy": 0.46156735473880167,
-      "eval_loss": 2.7969932556152344,
-      "eval_runtime": 72.5419,
-      "eval_samples_per_second": 836.771,
-      "eval_steps_per_second": 13.082,
+      "eval_accuracy": 0.4606821383794124,
+      "eval_loss": 2.8019886016845703,
+      "eval_runtime": 72.5563,
+      "eval_samples_per_second": 836.316,
+      "eval_steps_per_second": 13.079,
       "step": 27072
     },
     {
-      "epoch": 12.411347517730496,
-      "grad_norm": 0.18872858583927155,
-      "learning_rate": 0.00087478125,
-      "loss": 2.6167,
+      "epoch": 12.411484314377564,
+      "grad_norm": 0.1861707717180252,
+      "learning_rate": 0.0008748437500000001,
+      "loss": 2.6148,
       "step": 28000
     },
     {
-      "epoch": 12.854609929078014,
-      "grad_norm": 0.1738893836736679,
-      "learning_rate": 0.0009060312499999999,
-      "loss": 2.6375,
+      "epoch": 12.854894135905111,
+      "grad_norm": 0.18803346157073975,
+      "learning_rate": 0.0009060625,
+      "loss": 2.6384,
       "step": 29000
     },
     {
       "epoch": 13.0,
-      "eval_accuracy": 0.4621078374843455,
-      "eval_loss": 2.7915232181549072,
-      "eval_runtime": 72.7216,
-      "eval_samples_per_second": 834.704,
-      "eval_steps_per_second": 13.05,
+      "eval_accuracy": 0.46163558106169295,
+      "eval_loss": 2.7958271503448486,
+      "eval_runtime": 72.8339,
+      "eval_samples_per_second": 833.128,
+      "eval_steps_per_second": 13.03,
       "step": 29328
     },
     {
-      "epoch": 13.297872340425531,
-      "grad_norm": 0.1877707690000534,
-      "learning_rate": 0.00093725,
-      "loss": 2.6153,
+      "epoch": 13.297971400066512,
+      "grad_norm": 0.1761549860239029,
+      "learning_rate": 0.0009373125,
+      "loss": 2.6142,
       "step": 30000
     },
     {
-      "epoch": 13.74113475177305,
-      "grad_norm": 0.186727836728096,
-      "learning_rate": 0.0009685000000000001,
-      "loss": 2.6278,
+      "epoch": 13.741381221594057,
+      "grad_norm": 0.1844184547662735,
+      "learning_rate": 0.00096853125,
+      "loss": 2.6297,
       "step": 31000
     },
     {
       "epoch": 14.0,
-      "eval_accuracy": 0.462822946677559,
-      "eval_loss": 2.786189317703247,
-      "eval_runtime": 72.4588,
-      "eval_samples_per_second": 837.731,
-      "eval_steps_per_second": 13.097,
+      "eval_accuracy": 0.46196660074708856,
+      "eval_loss": 2.7939445972442627,
+      "eval_runtime": 72.6804,
+      "eval_samples_per_second": 834.888,
+      "eval_steps_per_second": 13.057,
       "step": 31584
     },
     {
-      "epoch": 14.184397163120567,
-      "grad_norm": 0.20270851254463196,
-      "learning_rate": 0.00099971875,
-      "loss": 2.6168,
+      "epoch": 14.18445848575546,
+      "grad_norm": 0.1886565387248993,
+      "learning_rate": 0.00099978125,
+      "loss": 2.6147,
       "step": 32000
     },
     {
-      "epoch": 14.627659574468085,
-      "grad_norm": 0.17972639203071594,
-      "learning_rate": 0.0009244664634146341,
-      "loss": 2.6102,
+      "epoch": 14.627868307283006,
+      "grad_norm": 0.17688792943954468,
+      "learning_rate": 0.0009241984732824427,
+      "loss": 2.612,
       "step": 33000
     },
     {
       "epoch": 15.0,
-      "eval_accuracy": 0.46529976086578023,
-      "eval_loss": 2.763315200805664,
-      "eval_runtime": 72.4263,
-      "eval_samples_per_second": 838.107,
-      "eval_steps_per_second": 13.103,
+      "eval_accuracy": 0.4653180942779221,
+      "eval_loss": 2.764906167984009,
+      "eval_runtime": 72.6592,
+      "eval_samples_per_second": 835.131,
+      "eval_steps_per_second": 13.061,
       "step": 33840
     },
     {
-      "epoch": 15.070921985815604,
-      "grad_norm": 0.17892582714557648,
-      "learning_rate": 0.0008483231707317073,
-      "loss": 2.6,
+      "epoch": 15.070945571444408,
+      "grad_norm": 0.19411760568618774,
+      "learning_rate": 0.00084793893129771,
+      "loss": 2.5952,
       "step": 34000
     },
     {
-      "epoch": 15.51418439716312,
-      "grad_norm": 0.16611941158771515,
-      "learning_rate": 0.0007721036585365854,
-      "loss": 2.5602,
+      "epoch": 15.514355392971954,
+      "grad_norm": 0.17588546872138977,
+      "learning_rate": 0.0007716793893129771,
+      "loss": 2.5635,
       "step": 35000
     },
     {
-      "epoch": 15.957446808510639,
-      "grad_norm": 0.1676749587059021,
-      "learning_rate": 0.0006960365853658537,
-      "loss": 2.5668,
+      "epoch": 15.957765214499501,
+      "grad_norm": 0.17366230487823486,
+      "learning_rate": 0.0006953435114503817,
+      "loss": 2.5667,
       "step": 36000
     },
     {
       "epoch": 16.0,
-      "eval_accuracy": 0.46856268478957125,
-      "eval_loss": 2.739426374435425,
-      "eval_runtime": 72.5666,
-      "eval_samples_per_second": 836.486,
-      "eval_steps_per_second": 13.078,
+      "eval_accuracy": 0.4685543578011297,
+      "eval_loss": 2.7425177097320557,
+      "eval_runtime": 72.8257,
+      "eval_samples_per_second": 833.222,
+      "eval_steps_per_second": 13.031,
       "step": 36096
     },
     {
-      "epoch": 16.400709219858157,
-      "grad_norm": 0.17558415234088898,
-      "learning_rate": 0.0006198170731707318,
-      "loss": 2.5115,
+      "epoch": 16.400842478660902,
+      "grad_norm": 0.18736566603183746,
+      "learning_rate": 0.0006190076335877863,
+      "loss": 2.5093,
       "step": 37000
     },
     {
-      "epoch": 16.843971631205672,
-      "grad_norm": 0.1874464899301529,
-      "learning_rate": 0.0005435975609756098,
-      "loss": 2.5178,
+      "epoch": 16.84425230018845,
+      "grad_norm": 0.18060249090194702,
+      "learning_rate": 0.0005426717557251909,
+      "loss": 2.5177,
       "step": 38000
     },
     {
       "epoch": 17.0,
-      "eval_accuracy": 0.47171248592021775,
-      "eval_loss": 2.7182633876800537,
-      "eval_runtime": 72.699,
-      "eval_samples_per_second": 834.964,
-      "eval_steps_per_second": 13.054,
+      "eval_accuracy": 0.4714397611384699,
+      "eval_loss": 2.7205777168273926,
+      "eval_runtime": 72.8204,
+      "eval_samples_per_second": 833.283,
+      "eval_steps_per_second": 13.032,
       "step": 38352
     },
     {
-      "epoch": 17.28723404255319,
-      "grad_norm": 0.18552443385124207,
-      "learning_rate": 0.00046745426829268295,
-      "loss": 2.4731,
+      "epoch": 17.28732956434985,
+      "grad_norm": 0.1875220090150833,
+      "learning_rate": 0.000466412213740458,
+      "loss": 2.4733,
       "step": 39000
     },
     {
-      "epoch": 17.73049645390071,
-      "grad_norm": 0.1822243332862854,
-      "learning_rate": 0.000391234756097561,
-      "loss": 2.462,
+      "epoch": 17.730739385877396,
+      "grad_norm": 0.18848279118537903,
+      "learning_rate": 0.00039007633587786263,
+      "loss": 2.4607,
       "step": 40000
     },
     {
       "epoch": 18.0,
-      "eval_accuracy": 0.4747758459901975,
-      "eval_loss": 2.697719097137451,
-      "eval_runtime": 72.2911,
-      "eval_samples_per_second": 839.675,
-      "eval_steps_per_second": 13.127,
+      "eval_accuracy": 0.47464791190042266,
+      "eval_loss": 2.699930429458618,
+      "eval_runtime": 72.4963,
+      "eval_samples_per_second": 837.008,
+      "eval_steps_per_second": 13.09,
       "step": 40608
     },
     {
-      "epoch": 18.173758865248228,
-      "grad_norm": 0.18331420421600342,
-      "learning_rate": 0.000315015243902439,
-      "loss": 2.4319,
+      "epoch": 18.1738166500388,
+      "grad_norm": 0.19309544563293457,
+      "learning_rate": 0.0003138167938931298,
+      "loss": 2.43,
       "step": 41000
     },
     {
-      "epoch": 18.617021276595743,
-      "grad_norm": 0.19214719533920288,
-      "learning_rate": 0.00023879573170731708,
-      "loss": 2.3974,
+      "epoch": 18.617226471566344,
+      "grad_norm": 0.1929185390472412,
+      "learning_rate": 0.00023748091603053434,
+      "loss": 2.397,
       "step": 42000
     },
     {
       "epoch": 19.0,
-      "eval_accuracy": 0.4773433651479076,
-      "eval_loss": 2.683954954147339,
-      "eval_runtime": 72.4404,
-      "eval_samples_per_second": 837.944,
-      "eval_steps_per_second": 13.1,
+      "eval_accuracy": 0.4773406620393708,
+      "eval_loss": 2.6864736080169678,
+      "eval_runtime": 72.792,
+      "eval_samples_per_second": 833.608,
+      "eval_steps_per_second": 13.037,
       "step": 42864
     },
     {
-      "epoch": 19.06028368794326,
-      "grad_norm": 0.19565586745738983,
-      "learning_rate": 0.00016265243902439025,
-      "loss": 2.3863,
+      "epoch": 19.060303735727746,
+      "grad_norm": 0.20593929290771484,
+      "learning_rate": 0.00016114503816793893,
+      "loss": 2.3837,
       "step": 43000
     },
     {
-      "epoch": 19.50354609929078,
-      "grad_norm": 0.19951286911964417,
-      "learning_rate": 8.643292682926828e-05,
-      "loss": 2.3273,
+      "epoch": 19.503713557255292,
+      "grad_norm": 0.2001897543668747,
+      "learning_rate": 8.480916030534351e-05,
+      "loss": 2.3276,
       "step": 44000
     },
     {
-      "epoch": 19.9468085106383,
-      "grad_norm": 0.19867576658725739,
-      "learning_rate": 1.0213414634146342e-05,
-      "loss": 2.3259,
+      "epoch": 19.94712337878284,
+      "grad_norm": 0.19933941960334778,
+      "learning_rate": 8.549618320610688e-06,
+      "loss": 2.3241,
       "step": 45000
     },
     {
-      "epoch": 20.0,
-      "eval_accuracy": 0.47877642614021604,
-      "eval_loss": 2.6820449829101562,
-      "eval_runtime": 72.6976,
-      "eval_samples_per_second": 834.98,
-      "eval_steps_per_second": 13.054,
-      "step": 45120
+      "epoch": 19.991464360935595,
+      "eval_accuracy": 0.47868057440510814,
+      "eval_loss": 2.6840312480926514,
+      "eval_runtime": 72.9711,
+      "eval_samples_per_second": 831.562,
+      "eval_steps_per_second": 13.005,
+      "step": 45100
     },
     {
-      "epoch": 20.0,
-      "step": 45120,
-      "total_flos": 1.50902942072832e+18,
-      "train_loss": 2.805498681169875,
-      "train_runtime": 30524.4807,
-      "train_samples_per_second": 378.402,
-      "train_steps_per_second": 1.478
+      "epoch": 19.991464360935595,
+      "step": 45100,
+      "total_flos": 1.507910045663232e+18,
+      "train_loss": 2.8050402250099604,
+      "train_runtime": 30336.2728,
+      "train_samples_per_second": 380.629,
+      "train_steps_per_second": 1.487
     }
   ],
   "logging_steps": 1000,
-  "max_steps": 45120,
+  "max_steps": 45100,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
   "save_steps": 500,
@@ -539,7 +539,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.50902942072832e+18,
+  "total_flos": 1.507910045663232e+18,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null