Training in progress, step 50, checkpoint

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d591479f455bcc3cd790651ad470e44e9d2cd5efdff513aa1389275e6e75f791
+oid sha256:0c4af77eee3e9f3b04e842b413ac6d5ace3e1651e0cc4444f52538cba9541deb
 size 295765866

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0599a104f0827658ba40405bb4115d7e8c2f7baef02dc9207433faa015dba80e
+oid sha256:fd26ef277c084f3f812a01bd19a5c765424fbc1c136168f9f472b93513ccd3ed
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3501b72d1929c3e111ccf4c6633317b43a8056f90d3818363f1d3d3b04b16dc0
+oid sha256:f964d487b74c19edf034a9714f2244bb673897970c7f365a947d46ffc5877189
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e05b88be4dd8576afce48f573920049ba07f7e65dd4ac3565dc6cd0958783897
+oid sha256:568ed7fb7a698efdf9d4434f019fb4ade5f34c6132c8564a0c8e54646daa62eb
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2df0662a6728a18b24ddad0bd9d7cbadff42f580ad952d26e63e2073da23dc78
+oid sha256:a570659a108dc2378f08985b122b038e52db4ab96e69e1cec920eb350a4ad6b0
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0478220dc1c61b501af7edfe8fa4a46b7ebe1a2bbd0ae4d7514712a0c8b72900
+oid sha256:3192059420211edc26748a07e2699e00ee3be65f92a2cf84ef56d22611d9eb5c
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": NaN,
   "best_model_checkpoint": "miner_id_24/checkpoint-25",
-  "epoch": 0.23460410557184752,
+  "epoch": 0.46920821114369504,
   "eval_steps": 25,
-  "global_step": 25,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -198,6 +198,189 @@
       "eval_samples_per_second": 98.829,
       "eval_steps_per_second": 6.194,
       "step": 25
+    },
+    {
+      "epoch": 0.2439882697947214,
+      "grad_norm": NaN,
+      "learning_rate": 8.842005554284296e-05,
+      "loss": 0.0,
+      "step": 26
+    },
+    {
+      "epoch": 0.2533724340175953,
+      "grad_norm": NaN,
+      "learning_rate": 8.73410738492077e-05,
+      "loss": 0.0,
+      "step": 27
+    },
+    {
+      "epoch": 0.2627565982404692,
+      "grad_norm": NaN,
+      "learning_rate": 8.622126023955446e-05,
+      "loss": 0.0,
+      "step": 28
+    },
+    {
+      "epoch": 0.2721407624633431,
+      "grad_norm": NaN,
+      "learning_rate": 8.506183921362443e-05,
+      "loss": 0.0,
+      "step": 29
+    },
+    {
+      "epoch": 0.28152492668621704,
+      "grad_norm": NaN,
+      "learning_rate": 8.386407858128706e-05,
+      "loss": 0.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.2909090909090909,
+      "grad_norm": NaN,
+      "learning_rate": 8.262928807620843e-05,
+      "loss": 0.0,
+      "step": 31
+    },
+    {
+      "epoch": 0.3002932551319648,
+      "grad_norm": NaN,
+      "learning_rate": 8.135881792367686e-05,
+      "loss": 0.0,
+      "step": 32
+    },
+    {
+      "epoch": 0.3096774193548387,
+      "grad_norm": NaN,
+      "learning_rate": 8.005405736415126e-05,
+      "loss": 0.0,
+      "step": 33
+    },
+    {
+      "epoch": 0.31906158357771264,
+      "grad_norm": NaN,
+      "learning_rate": 7.871643313414718e-05,
+      "loss": 0.0,
+      "step": 34
+    },
+    {
+      "epoch": 0.3284457478005865,
+      "grad_norm": NaN,
+      "learning_rate": 7.734740790612136e-05,
+      "loss": 0.0,
+      "step": 35
+    },
+    {
+      "epoch": 0.3378299120234604,
+      "grad_norm": NaN,
+      "learning_rate": 7.594847868906076e-05,
+      "loss": 0.0,
+      "step": 36
+    },
+    {
+      "epoch": 0.3472140762463343,
+      "grad_norm": NaN,
+      "learning_rate": 7.452117519152542e-05,
+      "loss": 0.0,
+      "step": 37
+    },
+    {
+      "epoch": 0.35659824046920824,
+      "grad_norm": NaN,
+      "learning_rate": 7.30670581489344e-05,
+      "loss": 0.0,
+      "step": 38
+    },
+    {
+      "epoch": 0.3659824046920821,
+      "grad_norm": NaN,
+      "learning_rate": 7.158771761692464e-05,
+      "loss": 0.0,
+      "step": 39
+    },
+    {
+      "epoch": 0.375366568914956,
+      "grad_norm": NaN,
+      "learning_rate": 7.008477123264848e-05,
+      "loss": 0.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.3847507331378299,
+      "grad_norm": NaN,
+      "learning_rate": 6.855986244591104e-05,
+      "loss": 0.0,
+      "step": 41
+    },
+    {
+      "epoch": 0.39413489736070384,
+      "grad_norm": NaN,
+      "learning_rate": 6.701465872208216e-05,
+      "loss": 0.0,
+      "step": 42
+    },
+    {
+      "epoch": 0.4035190615835777,
+      "grad_norm": NaN,
+      "learning_rate": 6.545084971874738e-05,
+      "loss": 0.0,
+      "step": 43
+    },
+    {
+      "epoch": 0.4129032258064516,
+      "grad_norm": NaN,
+      "learning_rate": 6.387014543809223e-05,
+      "loss": 0.0,
+      "step": 44
+    },
+    {
+      "epoch": 0.4222873900293255,
+      "grad_norm": NaN,
+      "learning_rate": 6.227427435703997e-05,
+      "loss": 0.0,
+      "step": 45
+    },
+    {
+      "epoch": 0.43167155425219944,
+      "grad_norm": NaN,
+      "learning_rate": 6.066498153718735e-05,
+      "loss": 0.0,
+      "step": 46
+    },
+    {
+      "epoch": 0.4410557184750733,
+      "grad_norm": NaN,
+      "learning_rate": 5.90440267166055e-05,
+      "loss": 0.0,
+      "step": 47
+    },
+    {
+      "epoch": 0.4504398826979472,
+      "grad_norm": NaN,
+      "learning_rate": 5.74131823855921e-05,
+      "loss": 0.0,
+      "step": 48
+    },
+    {
+      "epoch": 0.45982404692082113,
+      "grad_norm": NaN,
+      "learning_rate": 5.577423184847932e-05,
+      "loss": 0.0,
+      "step": 49
+    },
+    {
+      "epoch": 0.46920821114369504,
+      "grad_norm": NaN,
+      "learning_rate": 5.4128967273616625e-05,
+      "loss": 0.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.46920821114369504,
+      "eval_loss": NaN,
+      "eval_runtime": 14.5301,
+      "eval_samples_per_second": 98.829,
+      "eval_steps_per_second": 6.194,
+      "step": 50
     }
   ],
   "logging_steps": 1,
@@ -212,7 +395,7 @@
         "early_stopping_threshold": 0.0
       },
       "attributes": {
-        "early_stopping_patience_counter": 0
+        "early_stopping_patience_counter": 1
       }
     },
     "TrainerControl": {
@@ -221,12 +404,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 1.059536353886208e+17,
+  "total_flos": 2.119072707772416e+17,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null