Model save

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.992,
     "total_flos": 0.0,
-    "train_loss": 0.8412376757591001,
-    "train_runtime": 702.1132,
+    "train_loss": 0.8421394844208995,
+    "train_runtime": 736.5998,
     "train_samples": 2000,
-    "train_samples_per_second": 2.849,
-    "train_steps_per_second": 0.088
+    "train_samples_per_second": 2.715,
+    "train_steps_per_second": 0.084
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.992,
     "total_flos": 0.0,
-    "train_loss": 0.8412376757591001,
-    "train_runtime": 702.1132,
+    "train_loss": 0.8421394844208995,
+    "train_runtime": 736.5998,
     "train_samples": 2000,
-    "train_samples_per_second": 2.849,
-    "train_steps_per_second": 0.088
+    "train_samples_per_second": 2.715,
+    "train_steps_per_second": 0.084
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.016,
-      "grad_norm": 22.116561494440536,
+      "grad_norm": 22.116731061155882,
       "learning_rate": 7.142857142857142e-08,
       "logits/generated": -0.9303410053253174,
       "logits/real": -0.6472625732421875,
@@ -25,102 +25,102 @@
     },
     {
       "epoch": 0.16,
-      "grad_norm": 19.166988301842625,
+      "grad_norm": 19.00686173642431,
       "learning_rate": 4.727272727272727e-07,
-      "logits/generated": -0.7766941785812378,
-      "logits/real": -0.7624438405036926,
-      "logps/generated": -174.18157958984375,
-      "logps/real": -182.86370849609375,
-      "loss": 0.8573,
-      "rewards/accuracies": 0.5555555820465088,
-      "rewards/generated": 0.08411566913127899,
-      "rewards/margins": 0.02031405083835125,
-      "rewards/real": 0.1044297143816948,
+      "logits/generated": -0.7783747315406799,
+      "logits/real": -0.7638665437698364,
+      "logps/generated": -174.0995330810547,
+      "logps/real": -182.80723571777344,
+      "loss": 0.8563,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/generated": 0.09232226759195328,
+      "rewards/margins": 0.01775742694735527,
+      "rewards/real": 0.11007969081401825,
       "step": 10
     },
     {
       "epoch": 0.32,
-      "grad_norm": 20.3933839095788,
+      "grad_norm": 20.37082961992304,
       "learning_rate": 3.818181818181818e-07,
-      "logits/generated": -0.7564908862113953,
-      "logits/real": -0.7665221691131592,
-      "logps/generated": -163.1689910888672,
-      "logps/real": -164.83465576171875,
-      "loss": 0.8524,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/generated": 0.45903801918029785,
-      "rewards/margins": -0.008176992647349834,
-      "rewards/real": 0.45086097717285156,
+      "logits/generated": -0.7393258213996887,
+      "logits/real": -0.7510851621627808,
+      "logps/generated": -163.16494750976562,
+      "logps/real": -164.861572265625,
+      "loss": 0.8527,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/generated": 0.45944079756736755,
+      "rewards/margins": -0.011271441355347633,
+      "rewards/real": 0.44816938042640686,
       "step": 20
     },
     {
       "epoch": 0.48,
-      "grad_norm": 19.977285440131215,
+      "grad_norm": 19.828947810570654,
       "learning_rate": 2.909090909090909e-07,
-      "logits/generated": -0.8268370628356934,
-      "logits/real": -0.6981123685836792,
-      "logps/generated": -146.0032958984375,
-      "logps/real": -154.56153869628906,
-      "loss": 0.8307,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/generated": 0.660775363445282,
-      "rewards/margins": 0.10715095698833466,
-      "rewards/real": 0.7679262757301331,
+      "logits/generated": -0.8125241994857788,
+      "logits/real": -0.6840296983718872,
+      "logps/generated": -145.97573852539062,
+      "logps/real": -154.54061889648438,
+      "loss": 0.8316,
+      "rewards/accuracies": 0.625,
+      "rewards/generated": 0.6635308265686035,
+      "rewards/margins": 0.1064867228269577,
+      "rewards/real": 0.7700175046920776,
       "step": 30
     },
     {
       "epoch": 0.64,
-      "grad_norm": 19.443118690010486,
+      "grad_norm": 20.449526730384434,
       "learning_rate": 2e-07,
-      "logits/generated": -0.7490435242652893,
-      "logits/real": -0.6841712594032288,
-      "logps/generated": -165.5490264892578,
-      "logps/real": -177.98483276367188,
-      "loss": 0.8378,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/generated": 0.8964563608169556,
-      "rewards/margins": 0.039561666548252106,
-      "rewards/real": 0.9360179901123047,
+      "logits/generated": -0.7401232719421387,
+      "logits/real": -0.6730000972747803,
+      "logps/generated": -165.48190307617188,
+      "logps/real": -177.96424865722656,
+      "loss": 0.8405,
+      "rewards/accuracies": 0.5625,
+      "rewards/generated": 0.9031688570976257,
+      "rewards/margins": 0.03490729257464409,
+      "rewards/real": 0.9380761384963989,
       "step": 40
     },
     {
       "epoch": 0.8,
-      "grad_norm": 20.705818914648916,
+      "grad_norm": 20.801102601242686,
       "learning_rate": 1.0909090909090908e-07,
-      "logits/generated": -0.7975619435310364,
-      "logits/real": -0.6225983500480652,
-      "logps/generated": -163.07777404785156,
-      "logps/real": -180.90386962890625,
+      "logits/generated": -0.7878767251968384,
+      "logits/real": -0.6102726459503174,
+      "logps/generated": -163.09925842285156,
+      "logps/real": -180.8894500732422,
       "loss": 0.8413,
-      "rewards/accuracies": 0.625,
-      "rewards/generated": 1.017493486404419,
-      "rewards/margins": 0.0801156610250473,
-      "rewards/real": 1.097609043121338,
+      "rewards/accuracies": 0.5625,
+      "rewards/generated": 1.0153449773788452,
+      "rewards/margins": 0.08370596170425415,
+      "rewards/real": 1.0990509986877441,
       "step": 50
     },
     {
       "epoch": 0.96,
-      "grad_norm": 20.424310682494383,
+      "grad_norm": 20.460813305727022,
       "learning_rate": 1.818181818181818e-08,
-      "logits/generated": -0.6620000600814819,
-      "logits/real": -0.5432597398757935,
-      "logps/generated": -169.11434936523438,
-      "logps/real": -175.96197509765625,
-      "loss": 0.8309,
+      "logits/generated": -0.652385413646698,
+      "logits/real": -0.5340145230293274,
+      "logps/generated": -169.20877075195312,
+      "logps/real": -176.0380859375,
+      "loss": 0.8323,
       "rewards/accuracies": 0.5625,
-      "rewards/generated": 1.11982262134552,
-      "rewards/margins": 0.07809869945049286,
-      "rewards/real": 1.1979211568832397,
+      "rewards/generated": 1.1103785037994385,
+      "rewards/margins": 0.07993375509977341,
+      "rewards/real": 1.190312385559082,
       "step": 60
     },
     {
       "epoch": 0.992,
       "step": 62,
       "total_flos": 0.0,
-      "train_loss": 0.8412376757591001,
-      "train_runtime": 702.1132,
-      "train_samples_per_second": 2.849,
-      "train_steps_per_second": 0.088
+      "train_loss": 0.8421394844208995,
+      "train_runtime": 736.5998,
+      "train_samples_per_second": 2.715,
+      "train_steps_per_second": 0.084
     }
   ],
   "logging_steps": 10,