Upload 11 files

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:945f85af1cc3ec7873a9b404a49fb40e388f374c8a4d560f43c362f6b590ddf4
+oid sha256:65ebc8867eb68b43b53f8b168d233d581038889786395a1deeb0793e650370fc
 size 3416264

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71b88645a539bf5897c0921d75267259bc69d03af216626593b9748b36c6b945
+oid sha256:92b7686e8135c430264a435243d5d21ee10202f63beca9991fc2970ea91dd57f
 size 1807738

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a506954e524976b6241592040344d2e00e3c53142510c864270196f96c337e4b
+oid sha256:22d3783e7365b2512df7ed24791ee332b6535a1eea8b7980b6eed32ed57b4061
 size 1064

trainer_state.json

@@ -10,242 +10,242 @@
   "log_history": [
     {
       "epoch": 0.11851851851851852,
-      "grad_norm": 3.0074994564056396,
-      "learning_rate": 0.00011428571428571428,
-      "logits/chosen": 0.5537575483322144,
-      "logits/rejected": 0.5866800546646118,
-      "logps/chosen": -108.80691528320312,
-      "logps/rejected": -181.76271057128906,
-      "loss": 0.6897,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.010402859188616276,
-      "rewards/margins": 0.007031619548797607,
-      "rewards/rejected": -0.01743447780609131,
+      "grad_norm": 3.8725790977478027,
+      "learning_rate": 5.714285714285714e-05,
+      "logits/chosen": 0.5535547137260437,
+      "logits/rejected": 0.6790706515312195,
+      "logps/chosen": -154.02122497558594,
+      "logps/rejected": -190.33836364746094,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.28125,
+      "rewards/chosen": -0.00438268156722188,
+      "rewards/margins": 0.0031897304579615593,
+      "rewards/rejected": -0.0075724124908447266,
       "step": 4
     },
     {
       "epoch": 0.23703703703703705,
-      "grad_norm": 4.682945251464844,
-      "learning_rate": 0.00019661016949152545,
-      "logits/chosen": 0.6307014226913452,
-      "logits/rejected": 0.5614228248596191,
-      "logps/chosen": -125.5777816772461,
-      "logps/rejected": -216.0275421142578,
-      "loss": 0.6417,
+      "grad_norm": 4.261082649230957,
+      "learning_rate": 9.830508474576272e-05,
+      "logits/chosen": 0.8175854086875916,
+      "logits/rejected": 0.7639511823654175,
+      "logps/chosen": -176.618408203125,
+      "logps/rejected": -232.55316162109375,
+      "loss": 0.6661,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.12017861008644104,
-      "rewards/margins": 0.1225859746336937,
-      "rewards/rejected": -0.24276459217071533,
+      "rewards/chosen": -0.024643946439027786,
+      "rewards/margins": 0.05728544294834137,
+      "rewards/rejected": -0.08192938566207886,
       "step": 8
     },
     {
       "epoch": 0.35555555555555557,
-      "grad_norm": 4.1338911056518555,
-      "learning_rate": 0.00018305084745762714,
-      "logits/chosen": 0.5009933710098267,
-      "logits/rejected": 0.4711945056915283,
-      "logps/chosen": -128.8936767578125,
-      "logps/rejected": -248.692626953125,
-      "loss": 0.5783,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.552248477935791,
-      "rewards/margins": 0.34952110052108765,
-      "rewards/rejected": -0.9017695784568787,
+      "grad_norm": 3.7274258136749268,
+      "learning_rate": 9.152542372881357e-05,
+      "logits/chosen": 0.5340245366096497,
+      "logits/rejected": 0.748319685459137,
+      "logps/chosen": -137.61412048339844,
+      "logps/rejected": -184.79991149902344,
+      "loss": 0.6328,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.10764569044113159,
+      "rewards/margins": 0.14540965855121613,
+      "rewards/rejected": -0.2530553638935089,
       "step": 12
     },
     {
       "epoch": 0.4740740740740741,
-      "grad_norm": 3.1105973720550537,
-      "learning_rate": 0.00016949152542372882,
-      "logits/chosen": 0.3637569844722748,
-      "logits/rejected": 0.07950417697429657,
-      "logps/chosen": -128.09384155273438,
-      "logps/rejected": -208.0956268310547,
-      "loss": 0.3987,
+      "grad_norm": 4.1018290519714355,
+      "learning_rate": 8.474576271186441e-05,
+      "logits/chosen": 0.43400460481643677,
+      "logits/rejected": 0.4534505605697632,
+      "logps/chosen": -113.1705551147461,
+      "logps/rejected": -201.7666778564453,
+      "loss": 0.5399,
       "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.9156935214996338,
-      "rewards/margins": 1.3323218822479248,
-      "rewards/rejected": -2.2480154037475586,
+      "rewards/chosen": -0.11035704612731934,
+      "rewards/margins": 0.3914012014865875,
+      "rewards/rejected": -0.5017582178115845,
       "step": 16
     },
     {
       "epoch": 0.5925925925925926,
-      "grad_norm": 5.975098133087158,
-      "learning_rate": 0.00015593220338983051,
-      "logits/chosen": 0.1583789736032486,
-      "logits/rejected": -0.2883496880531311,
-      "logps/chosen": -135.5908966064453,
-      "logps/rejected": -229.41116333007812,
-      "loss": 0.5083,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -2.0393593311309814,
-      "rewards/margins": 2.160950183868408,
-      "rewards/rejected": -4.200309753417969,
+      "grad_norm": 7.585125923156738,
+      "learning_rate": 7.796610169491526e-05,
+      "logits/chosen": 0.3885829448699951,
+      "logits/rejected": 0.4270927906036377,
+      "logps/chosen": -162.44253540039062,
+      "logps/rejected": -242.7151336669922,
+      "loss": 0.5997,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5063179731369019,
+      "rewards/margins": 0.40314924716949463,
+      "rewards/rejected": -0.9094672799110413,
       "step": 20
     },
     {
       "epoch": 0.7111111111111111,
-      "grad_norm": 3.235316514968872,
-      "learning_rate": 0.0001423728813559322,
-      "logits/chosen": 0.3243708610534668,
-      "logits/rejected": -0.5372164845466614,
-      "logps/chosen": -159.70765686035156,
-      "logps/rejected": -289.2506408691406,
-      "loss": 0.2323,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -2.4557015895843506,
-      "rewards/margins": 3.490675687789917,
-      "rewards/rejected": -5.946376800537109,
+      "grad_norm": 4.802177429199219,
+      "learning_rate": 7.11864406779661e-05,
+      "logits/chosen": 0.39339229464530945,
+      "logits/rejected": 0.1176382452249527,
+      "logps/chosen": -157.732177734375,
+      "logps/rejected": -208.583251953125,
+      "loss": 0.4315,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.5111438632011414,
+      "rewards/margins": 0.9935877323150635,
+      "rewards/rejected": -1.5047316551208496,
       "step": 24
     },
     {
       "epoch": 0.8296296296296296,
-      "grad_norm": 6.991959095001221,
-      "learning_rate": 0.0001288135593220339,
-      "logits/chosen": 0.3035649359226227,
-      "logits/rejected": -0.3758375346660614,
-      "logps/chosen": -114.1632080078125,
-      "logps/rejected": -284.4447937011719,
-      "loss": 0.3505,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.068457841873169,
-      "rewards/margins": 2.7921276092529297,
-      "rewards/rejected": -4.8605852127075195,
+      "grad_norm": 8.156561851501465,
+      "learning_rate": 6.440677966101695e-05,
+      "logits/chosen": 0.011842329055070877,
+      "logits/rejected": 0.35012856125831604,
+      "logps/chosen": -158.82598876953125,
+      "logps/rejected": -232.102783203125,
+      "loss": 0.6494,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8290945291519165,
+      "rewards/margins": 0.6025375723838806,
+      "rewards/rejected": -1.4316319227218628,
       "step": 28
     },
     {
       "epoch": 0.9481481481481482,
-      "grad_norm": 5.21298885345459,
-      "learning_rate": 0.0001152542372881356,
-      "logits/chosen": 0.21702611446380615,
-      "logits/rejected": -0.6023719310760498,
-      "logps/chosen": -137.3132781982422,
-      "logps/rejected": -253.89906311035156,
-      "loss": 0.1907,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -2.5543203353881836,
-      "rewards/margins": 3.5818207263946533,
-      "rewards/rejected": -6.136141300201416,
+      "grad_norm": 6.632073402404785,
+      "learning_rate": 5.76271186440678e-05,
+      "logits/chosen": -0.034538522362709045,
+      "logits/rejected": 0.06912083178758621,
+      "logps/chosen": -163.13931274414062,
+      "logps/rejected": -159.48757934570312,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0138874053955078,
+      "rewards/margins": 0.7364886403083801,
+      "rewards/rejected": -1.7503761053085327,
       "step": 32
     },
     {
       "epoch": 1.0666666666666667,
-      "grad_norm": 3.979855537414551,
-      "learning_rate": 0.00010169491525423729,
-      "logits/chosen": -0.05026464909315109,
-      "logits/rejected": -0.6121890544891357,
-      "logps/chosen": -156.1046142578125,
-      "logps/rejected": -292.4217224121094,
-      "loss": 0.1849,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -3.4322896003723145,
-      "rewards/margins": 4.121622562408447,
-      "rewards/rejected": -7.5539116859436035,
+      "grad_norm": 4.534868240356445,
+      "learning_rate": 5.0847457627118643e-05,
+      "logits/chosen": 0.30354946851730347,
+      "logits/rejected": 0.13392731547355652,
+      "logps/chosen": -154.13514709472656,
+      "logps/rejected": -247.21238708496094,
+      "loss": 0.4295,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.7905898094177246,
+      "rewards/margins": 1.3606148958206177,
+      "rewards/rejected": -2.1512045860290527,
       "step": 36
     },
     {
       "epoch": 1.1851851851851851,
-      "grad_norm": 12.019647598266602,
-      "learning_rate": 8.813559322033899e-05,
-      "logits/chosen": -0.03930215910077095,
-      "logits/rejected": -0.899419367313385,
-      "logps/chosen": -173.2172088623047,
-      "logps/rejected": -324.8526916503906,
-      "loss": 0.1606,
-      "rewards/accuracies": 0.96875,
-      "rewards/chosen": -5.0753374099731445,
-      "rewards/margins": 6.455450534820557,
-      "rewards/rejected": -11.53078842163086,
+      "grad_norm": 8.053945541381836,
+      "learning_rate": 4.4067796610169495e-05,
+      "logits/chosen": 0.31644004583358765,
+      "logits/rejected": 0.34593597054481506,
+      "logps/chosen": -198.6423797607422,
+      "logps/rejected": -201.91390991210938,
+      "loss": 0.4743,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.8822494149208069,
+      "rewards/margins": 0.9695678949356079,
+      "rewards/rejected": -1.8518173694610596,
       "step": 40
     },
     {
       "epoch": 1.3037037037037038,
-      "grad_norm": 37.088966369628906,
-      "learning_rate": 7.457627118644068e-05,
-      "logits/chosen": 0.08398707211017609,
-      "logits/rejected": -0.9724540710449219,
-      "logps/chosen": -161.26870727539062,
-      "logps/rejected": -298.8813171386719,
-      "loss": 0.5551,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.843762397766113,
-      "rewards/margins": 5.168354034423828,
-      "rewards/rejected": -10.012115478515625,
+      "grad_norm": 3.4382383823394775,
+      "learning_rate": 3.728813559322034e-05,
+      "logits/chosen": 0.21577686071395874,
+      "logits/rejected": 0.19070810079574585,
+      "logps/chosen": -151.60324096679688,
+      "logps/rejected": -234.2410888671875,
+      "loss": 0.3084,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.48244649171829224,
+      "rewards/margins": 1.5388765335083008,
+      "rewards/rejected": -2.0213229656219482,
       "step": 44
     },
     {
       "epoch": 1.4222222222222223,
-      "grad_norm": 2.6414237022399902,
-      "learning_rate": 6.101694915254238e-05,
-      "logits/chosen": 0.19181180000305176,
-      "logits/rejected": -0.8720277547836304,
-      "logps/chosen": -156.1532745361328,
-      "logps/rejected": -357.791748046875,
-      "loss": 0.1984,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.283871650695801,
-      "rewards/margins": 7.30339241027832,
-      "rewards/rejected": -11.587264060974121,
+      "grad_norm": 3.3008296489715576,
+      "learning_rate": 3.050847457627119e-05,
+      "logits/chosen": 0.20948533713817596,
+      "logits/rejected": 0.24637725949287415,
+      "logps/chosen": -140.76052856445312,
+      "logps/rejected": -215.05499267578125,
+      "loss": 0.2868,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.4717501699924469,
+      "rewards/margins": 1.771106243133545,
+      "rewards/rejected": -2.242856502532959,
       "step": 48
     },
     {
       "epoch": 1.5407407407407407,
-      "grad_norm": 0.9389731287956238,
-      "learning_rate": 4.745762711864407e-05,
-      "logits/chosen": 0.09345383197069168,
-      "logits/rejected": -0.7088485956192017,
-      "logps/chosen": -148.07276916503906,
-      "logps/rejected": -283.626220703125,
-      "loss": 0.1336,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -4.233022212982178,
-      "rewards/margins": 5.9999284744262695,
-      "rewards/rejected": -10.232950210571289,
+      "grad_norm": 11.502059936523438,
+      "learning_rate": 2.3728813559322036e-05,
+      "logits/chosen": 0.08824478089809418,
+      "logits/rejected": -0.027387090027332306,
+      "logps/chosen": -148.2930450439453,
+      "logps/rejected": -225.59078979492188,
+      "loss": 0.5723,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.0404824018478394,
+      "rewards/margins": 1.086774230003357,
+      "rewards/rejected": -2.1272566318511963,
       "step": 52
     },
     {
       "epoch": 1.6592592592592592,
-      "grad_norm": 13.57490062713623,
-      "learning_rate": 3.389830508474576e-05,
-      "logits/chosen": 0.027704844251275063,
-      "logits/rejected": -0.7763456106185913,
-      "logps/chosen": -175.2841033935547,
-      "logps/rejected": -274.7032165527344,
-      "loss": 0.2452,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -4.180891513824463,
-      "rewards/margins": 5.651626110076904,
-      "rewards/rejected": -9.832517623901367,
+      "grad_norm": 4.403205394744873,
+      "learning_rate": 1.694915254237288e-05,
+      "logits/chosen": 0.2315160483121872,
+      "logits/rejected": 0.18600693345069885,
+      "logps/chosen": -134.02783203125,
+      "logps/rejected": -226.99459838867188,
+      "loss": 0.2737,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.5924757719039917,
+      "rewards/margins": 1.9642682075500488,
+      "rewards/rejected": -2.55674409866333,
       "step": 56
     },
     {
       "epoch": 1.7777777777777777,
-      "grad_norm": 3.9768831729888916,
-      "learning_rate": 2.033898305084746e-05,
-      "logits/chosen": 0.01869223639369011,
-      "logits/rejected": -0.6138023138046265,
-      "logps/chosen": -173.04226684570312,
-      "logps/rejected": -320.1624755859375,
-      "loss": 0.3316,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": -3.95318603515625,
-      "rewards/margins": 5.4156084060668945,
-      "rewards/rejected": -9.368793487548828,
+      "grad_norm": 4.00092077255249,
+      "learning_rate": 1.016949152542373e-05,
+      "logits/chosen": 0.10627911239862442,
+      "logits/rejected": 0.08208482712507248,
+      "logps/chosen": -172.69593811035156,
+      "logps/rejected": -211.60107421875,
+      "loss": 0.3421,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.770930826663971,
+      "rewards/margins": 1.9104766845703125,
+      "rewards/rejected": -2.6814074516296387,
       "step": 60
     },
     {
       "epoch": 1.8962962962962964,
-      "grad_norm": 1.7069957256317139,
-      "learning_rate": 6.779661016949153e-06,
-      "logits/chosen": 0.5393208861351013,
-      "logits/rejected": -1.024760365486145,
-      "logps/chosen": -110.98735046386719,
-      "logps/rejected": -315.8761291503906,
-      "loss": 0.0975,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": -3.0128798484802246,
-      "rewards/margins": 7.370530128479004,
-      "rewards/rejected": -10.383410453796387,
+      "grad_norm": 4.906458854675293,
+      "learning_rate": 3.3898305084745763e-06,
+      "logits/chosen": -0.028121717274188995,
+      "logits/rejected": 0.011860378086566925,
+      "logps/chosen": -121.95062255859375,
+      "logps/rejected": -200.142578125,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.47138434648513794,
+      "rewards/margins": 1.7951738834381104,
+      "rewards/rejected": -2.2665581703186035,
       "step": 64
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:601c68018ab495fa93d48c5d64b1832edfaac17e1bf33a15b98c4d63b583d330
+oid sha256:9a532c86fda2fd7158aecbe3b2707d774f9c356acd0b2658b92c3da108c9a58e
 size 5112