End of training

README.md

@@ -2,11 +2,23 @@
 library_name: transformers
 tags:
 - generated_from_trainer
+datasets:
+- kanishka/babylm2-rewritten-clean-spacy
 metrics:
 - accuracy
 model-index:
 - name: opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-1024_1e-3
-  results: []
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: kanishka/babylm2-rewritten-clean-spacy
+      type: kanishka/babylm2-rewritten-clean-spacy
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.4786608631587111
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -14,7 +26,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-1024_1e-3
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the kanishka/babylm2-rewritten-clean-spacy dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.6845
 - Accuracy: 0.4787

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 19.991464360935595,
+    "eval_accuracy": 0.4786608631587111,
+    "eval_loss": 2.684520721435547,
+    "eval_runtime": 71.5395,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 848.202,
+    "eval_steps_per_second": 13.265,
+    "perplexity": 14.651177701831413,
+    "total_flos": 1.507910045663232e+18,
+    "train_loss": 2.8046950542847493,
+    "train_runtime": 30489.2738,
+    "train_samples": 577344,
+    "train_samples_per_second": 378.719,
+    "train_steps_per_second": 1.479
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 19.991464360935595,
+    "eval_accuracy": 0.4786608631587111,
+    "eval_loss": 2.684520721435547,
+    "eval_runtime": 71.5395,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 848.202,
+    "eval_steps_per_second": 13.265,
+    "perplexity": 14.651177701831413
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 19.991464360935595,
+    "total_flos": 1.507910045663232e+18,
+    "train_loss": 2.8046950542847493,
+    "train_runtime": 30489.2738,
+    "train_samples": 577344,
+    "train_samples_per_second": 378.719,
+    "train_steps_per_second": 1.479
+}

trainer_state.json

@@ -0,0 +1,546 @@
+{
+  "best_metric": 2.684520721435547,
+  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-1024_1e-3/checkpoint-45100",
+  "epoch": 19.991464360935595,
+  "eval_steps": 500,
+  "global_step": 45100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4434098215275468,
+      "grad_norm": 0.5025885105133057,
+      "learning_rate": 3.125e-05,
+      "loss": 5.5904,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8868196430550936,
+      "grad_norm": 0.6121127009391785,
+      "learning_rate": 6.25e-05,
+      "loss": 4.093,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.3607157444388434,
+      "eval_loss": 3.8133068084716797,
+      "eval_runtime": 74.1752,
+      "eval_samples_per_second": 818.063,
+      "eval_steps_per_second": 12.794,
+      "step": 2256
+    },
+    {
+      "epoch": 1.3298969072164948,
+      "grad_norm": 0.6198744177818298,
+      "learning_rate": 9.375e-05,
+      "loss": 3.6959,
+      "step": 3000
+    },
+    {
+      "epoch": 1.7733067287440418,
+      "grad_norm": 0.6029963493347168,
+      "learning_rate": 0.000125,
+      "loss": 3.4493,
+      "step": 4000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.4091427869763594,
+      "eval_loss": 3.305894136428833,
+      "eval_runtime": 73.8861,
+      "eval_samples_per_second": 821.264,
+      "eval_steps_per_second": 12.844,
+      "step": 4512
+    },
+    {
+      "epoch": 2.2163839929054427,
+      "grad_norm": 0.5063702464103699,
+      "learning_rate": 0.00015625,
+      "loss": 3.2517,
+      "step": 5000
+    },
+    {
+      "epoch": 2.6597938144329896,
+      "grad_norm": 0.44880211353302,
+      "learning_rate": 0.0001875,
+      "loss": 3.132,
+      "step": 6000
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.4294944873137126,
+      "eval_loss": 3.098858594894409,
+      "eval_runtime": 73.6634,
+      "eval_samples_per_second": 823.747,
+      "eval_steps_per_second": 12.883,
+      "step": 6768
+    },
+    {
+      "epoch": 3.102871078594391,
+      "grad_norm": 0.4234854280948639,
+      "learning_rate": 0.00021875,
+      "loss": 3.0361,
+      "step": 7000
+    },
+    {
+      "epoch": 3.5462809001219378,
+      "grad_norm": 0.3789021968841553,
+      "learning_rate": 0.00025,
+      "loss": 2.9687,
+      "step": 8000
+    },
+    {
+      "epoch": 3.9896907216494846,
+      "grad_norm": 0.3866041898727417,
+      "learning_rate": 0.00028125000000000003,
+      "loss": 2.9232,
+      "step": 9000
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.44013804335181667,
+      "eval_loss": 2.991689920425415,
+      "eval_runtime": 72.9233,
+      "eval_samples_per_second": 832.108,
+      "eval_steps_per_second": 13.014,
+      "step": 9024
+    },
+    {
+      "epoch": 4.4327679858108855,
+      "grad_norm": 0.3524133861064911,
+      "learning_rate": 0.0003125,
+      "loss": 2.8604,
+      "step": 10000
+    },
+    {
+      "epoch": 4.876177807338433,
+      "grad_norm": 0.33133095502853394,
+      "learning_rate": 0.00034375,
+      "loss": 2.8445,
+      "step": 11000
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.446582457636977,
+      "eval_loss": 2.926978349685669,
+      "eval_runtime": 72.9486,
+      "eval_samples_per_second": 831.819,
+      "eval_steps_per_second": 13.009,
+      "step": 11280
+    },
+    {
+      "epoch": 5.319255071499834,
+      "grad_norm": 0.3199477791786194,
+      "learning_rate": 0.000375,
+      "loss": 2.797,
+      "step": 12000
+    },
+    {
+      "epoch": 5.762664893027381,
+      "grad_norm": 0.31949880719184875,
+      "learning_rate": 0.00040625000000000004,
+      "loss": 2.7872,
+      "step": 13000
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.45104353277237064,
+      "eval_loss": 2.8861489295959473,
+      "eval_runtime": 72.4638,
+      "eval_samples_per_second": 837.384,
+      "eval_steps_per_second": 13.096,
+      "step": 13536
+    },
+    {
+      "epoch": 6.205742157188782,
+      "grad_norm": 0.309347003698349,
+      "learning_rate": 0.0004375,
+      "loss": 2.7558,
+      "step": 14000
+    },
+    {
+      "epoch": 6.649151978716328,
+      "grad_norm": 0.3025387227535248,
+      "learning_rate": 0.00046871875,
+      "loss": 2.7438,
+      "step": 15000
+    },
+    {
+      "epoch": 7.0,
+      "eval_accuracy": 0.4540791293445526,
+      "eval_loss": 2.863698720932007,
+      "eval_runtime": 72.5858,
+      "eval_samples_per_second": 835.976,
+      "eval_steps_per_second": 13.074,
+      "step": 15792
+    },
+    {
+      "epoch": 7.09222924287773,
+      "grad_norm": 0.2948712706565857,
+      "learning_rate": 0.0004999375,
+      "loss": 2.7284,
+      "step": 16000
+    },
+    {
+      "epoch": 7.535639064405276,
+      "grad_norm": 0.25841575860977173,
+      "learning_rate": 0.0005311875000000001,
+      "loss": 2.7039,
+      "step": 17000
+    },
+    {
+      "epoch": 7.979048885932824,
+      "grad_norm": 0.25088393688201904,
+      "learning_rate": 0.0005624375,
+      "loss": 2.7139,
+      "step": 18000
+    },
+    {
+      "epoch": 8.0,
+      "eval_accuracy": 0.45652539196298164,
+      "eval_loss": 2.8414440155029297,
+      "eval_runtime": 72.7369,
+      "eval_samples_per_second": 834.239,
+      "eval_steps_per_second": 13.047,
+      "step": 18048
+    },
+    {
+      "epoch": 8.422126150094225,
+      "grad_norm": 0.262021005153656,
+      "learning_rate": 0.0005936875,
+      "loss": 2.6715,
+      "step": 19000
+    },
+    {
+      "epoch": 8.865535971621771,
+      "grad_norm": 0.2272481620311737,
+      "learning_rate": 0.00062490625,
+      "loss": 2.6887,
+      "step": 20000
+    },
+    {
+      "epoch": 9.0,
+      "eval_accuracy": 0.4580181472720928,
+      "eval_loss": 2.827446222305298,
+      "eval_runtime": 72.364,
+      "eval_samples_per_second": 838.538,
+      "eval_steps_per_second": 13.114,
+      "step": 20304
+    },
+    {
+      "epoch": 9.308613235783172,
+      "grad_norm": 0.23011185228824615,
+      "learning_rate": 0.0006561562500000001,
+      "loss": 2.6592,
+      "step": 21000
+    },
+    {
+      "epoch": 9.75202305731072,
+      "grad_norm": 0.21984772384166718,
+      "learning_rate": 0.0006873749999999999,
+      "loss": 2.6686,
+      "step": 22000
+    },
+    {
+      "epoch": 10.0,
+      "eval_accuracy": 0.45902574741168717,
+      "eval_loss": 2.8194234371185303,
+      "eval_runtime": 72.7839,
+      "eval_samples_per_second": 833.701,
+      "eval_steps_per_second": 13.039,
+      "step": 22560
+    },
+    {
+      "epoch": 10.19510032147212,
+      "grad_norm": 0.2220451980829239,
+      "learning_rate": 0.000718625,
+      "loss": 2.6496,
+      "step": 23000
+    },
+    {
+      "epoch": 10.638510142999667,
+      "grad_norm": 0.2019374966621399,
+      "learning_rate": 0.0007498437500000001,
+      "loss": 2.65,
+      "step": 24000
+    },
+    {
+      "epoch": 11.0,
+      "eval_accuracy": 0.4605366629182985,
+      "eval_loss": 2.8065686225891113,
+      "eval_runtime": 73.4344,
+      "eval_samples_per_second": 826.316,
+      "eval_steps_per_second": 12.923,
+      "step": 24816
+    },
+    {
+      "epoch": 11.081587407161068,
+      "grad_norm": 0.20551519095897675,
+      "learning_rate": 0.00078109375,
+      "loss": 2.6472,
+      "step": 25000
+    },
+    {
+      "epoch": 11.524997228688616,
+      "grad_norm": 0.19299697875976562,
+      "learning_rate": 0.00081234375,
+      "loss": 2.6326,
+      "step": 26000
+    },
+    {
+      "epoch": 11.968407050216163,
+      "grad_norm": 0.19943425059318542,
+      "learning_rate": 0.0008435625,
+      "loss": 2.6481,
+      "step": 27000
+    },
+    {
+      "epoch": 12.0,
+      "eval_accuracy": 0.4610733904636344,
+      "eval_loss": 2.8002209663391113,
+      "eval_runtime": 73.1449,
+      "eval_samples_per_second": 829.586,
+      "eval_steps_per_second": 12.974,
+      "step": 27072
+    },
+    {
+      "epoch": 12.411484314377564,
+      "grad_norm": 0.2178070843219757,
+      "learning_rate": 0.0008748125,
+      "loss": 2.6134,
+      "step": 28000
+    },
+    {
+      "epoch": 12.854894135905111,
+      "grad_norm": 0.18575814366340637,
+      "learning_rate": 0.0009060312499999999,
+      "loss": 2.6382,
+      "step": 29000
+    },
+    {
+      "epoch": 13.0,
+      "eval_accuracy": 0.46178842400506676,
+      "eval_loss": 2.7956042289733887,
+      "eval_runtime": 73.5828,
+      "eval_samples_per_second": 824.649,
+      "eval_steps_per_second": 12.897,
+      "step": 29328
+    },
+    {
+      "epoch": 13.297971400066512,
+      "grad_norm": 0.19861382246017456,
+      "learning_rate": 0.00093728125,
+      "loss": 2.6135,
+      "step": 30000
+    },
+    {
+      "epoch": 13.741381221594057,
+      "grad_norm": 0.1728564202785492,
+      "learning_rate": 0.0009685000000000001,
+      "loss": 2.6285,
+      "step": 31000
+    },
+    {
+      "epoch": 14.0,
+      "eval_accuracy": 0.4619170318094278,
+      "eval_loss": 2.7967946529388428,
+      "eval_runtime": 73.667,
+      "eval_samples_per_second": 823.707,
+      "eval_steps_per_second": 12.882,
+      "step": 31584
+    },
+    {
+      "epoch": 14.18445848575546,
+      "grad_norm": 0.18521134555339813,
+      "learning_rate": 0.00099975,
+      "loss": 2.6149,
+      "step": 32000
+    },
+    {
+      "epoch": 14.627868307283006,
+      "grad_norm": 0.17586995661258698,
+      "learning_rate": 0.0009242748091603054,
+      "loss": 2.6116,
+      "step": 33000
+    },
+    {
+      "epoch": 15.0,
+      "eval_accuracy": 0.4650235242416017,
+      "eval_loss": 2.7692339420318604,
+      "eval_runtime": 73.4533,
+      "eval_samples_per_second": 826.103,
+      "eval_steps_per_second": 12.92,
+      "step": 33840
+    },
+    {
+      "epoch": 15.070945571444408,
+      "grad_norm": 0.17404624819755554,
+      "learning_rate": 0.0008480152671755725,
+      "loss": 2.5972,
+      "step": 34000
+    },
+    {
+      "epoch": 15.514355392971954,
+      "grad_norm": 0.16976499557495117,
+      "learning_rate": 0.0007716793893129771,
+      "loss": 2.5609,
+      "step": 35000
+    },
+    {
+      "epoch": 15.957765214499501,
+      "grad_norm": 0.17162802815437317,
+      "learning_rate": 0.0006954198473282442,
+      "loss": 2.5677,
+      "step": 36000
+    },
+    {
+      "epoch": 16.0,
+      "eval_accuracy": 0.46843550867941114,
+      "eval_loss": 2.7445576190948486,
+      "eval_runtime": 72.7368,
+      "eval_samples_per_second": 834.241,
+      "eval_steps_per_second": 13.047,
+      "step": 36096
+    },
+    {
+      "epoch": 16.400842478660902,
+      "grad_norm": 0.17722822725772858,
+      "learning_rate": 0.0006190839694656488,
+      "loss": 2.508,
+      "step": 37000
+    },
+    {
+      "epoch": 16.84425230018845,
+      "grad_norm": 0.17358119785785675,
+      "learning_rate": 0.0005428244274809161,
+      "loss": 2.5197,
+      "step": 38000
+    },
+    {
+      "epoch": 17.0,
+      "eval_accuracy": 0.4717349128181266,
+      "eval_loss": 2.7191340923309326,
+      "eval_runtime": 72.6434,
+      "eval_samples_per_second": 835.313,
+      "eval_steps_per_second": 13.064,
+      "step": 38352
+    },
+    {
+      "epoch": 17.28732956434985,
+      "grad_norm": 0.18952704966068268,
+      "learning_rate": 0.0004664885496183206,
+      "loss": 2.4719,
+      "step": 39000
+    },
+    {
+      "epoch": 17.730739385877396,
+      "grad_norm": 0.1840035766363144,
+      "learning_rate": 0.0003902290076335878,
+      "loss": 2.462,
+      "step": 40000
+    },
+    {
+      "epoch": 18.0,
+      "eval_accuracy": 0.4746881745447025,
+      "eval_loss": 2.70100736618042,
+      "eval_runtime": 72.7995,
+      "eval_samples_per_second": 833.522,
+      "eval_steps_per_second": 13.036,
+      "step": 40608
+    },
+    {
+      "epoch": 18.1738166500388,
+      "grad_norm": 0.19474458694458008,
+      "learning_rate": 0.0003138931297709924,
+      "loss": 2.4276,
+      "step": 41000
+    },
+    {
+      "epoch": 18.617226471566344,
+      "grad_norm": 0.1999412178993225,
+      "learning_rate": 0.00023763358778625953,
+      "loss": 2.3973,
+      "step": 42000
+    },
+    {
+      "epoch": 19.0,
+      "eval_accuracy": 0.4773363320278672,
+      "eval_loss": 2.6860384941101074,
+      "eval_runtime": 72.6951,
+      "eval_samples_per_second": 834.719,
+      "eval_steps_per_second": 13.055,
+      "step": 42864
+    },
+    {
+      "epoch": 19.060303735727746,
+      "grad_norm": 0.2043563723564148,
+      "learning_rate": 0.00016129770992366412,
+      "loss": 2.3854,
+      "step": 43000
+    },
+    {
+      "epoch": 19.503713557255292,
+      "grad_norm": 0.2020735889673233,
+      "learning_rate": 8.50381679389313e-05,
+      "loss": 2.3263,
+      "step": 44000
+    },
+    {
+      "epoch": 19.94712337878284,
+      "grad_norm": 0.20228838920593262,
+      "learning_rate": 8.702290076335879e-06,
+      "loss": 2.324,
+      "step": 45000
+    },
+    {
+      "epoch": 19.991464360935595,
+      "eval_accuracy": 0.4786608631587111,
+      "eval_loss": 2.684520721435547,
+      "eval_runtime": 72.779,
+      "eval_samples_per_second": 833.757,
+      "eval_steps_per_second": 13.039,
+      "step": 45100
+    },
+    {
+      "epoch": 19.991464360935595,
+      "step": 45100,
+      "total_flos": 1.507910045663232e+18,
+      "train_loss": 2.8046950542847493,
+      "train_runtime": 30489.2738,
+      "train_samples_per_second": 378.719,
+      "train_steps_per_second": 1.479
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 45100,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.507910045663232e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}