| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 205, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.69230651855469, | |
| "epoch": 0.004878048780487805, | |
| "grad_norm": 1.988409597361304, | |
| "kl": 0.0002803802490234375, | |
| "learning_rate": 5e-08, | |
| "loss": -0.0332, | |
| "reward": 0.38933777809143066, | |
| "reward_std": 0.2539410889148712, | |
| "rewards/format_reward": 0.11098899692296982, | |
| "rewards/judgement_reward": 0.26093122363090515, | |
| "rewards/token_reward": 0.017417579889297485, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.14835357666016, | |
| "epoch": 0.00975609756097561, | |
| "grad_norm": 2.131261717353683, | |
| "kl": 0.000675201416015625, | |
| "learning_rate": 1e-07, | |
| "loss": -0.0789, | |
| "reward": 0.3299258053302765, | |
| "reward_std": 0.24830038845539093, | |
| "rewards/format_reward": 0.09890110045671463, | |
| "rewards/judgement_reward": 0.2083873748779297, | |
| "rewards/token_reward": 0.02263736166059971, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.86813354492188, | |
| "epoch": 0.014634146341463415, | |
| "grad_norm": 1.9020051525729609, | |
| "kl": 0.00048828125, | |
| "learning_rate": 1.5e-07, | |
| "loss": -0.0115, | |
| "reward": 0.3796335756778717, | |
| "reward_std": 0.2533051073551178, | |
| "rewards/format_reward": 0.09340659528970718, | |
| "rewards/judgement_reward": 0.26880943775177, | |
| "rewards/token_reward": 0.017417583614587784, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.22527313232422, | |
| "epoch": 0.01951219512195122, | |
| "grad_norm": 1.926669794956742, | |
| "kl": 0.0002899169921875, | |
| "learning_rate": 2e-07, | |
| "loss": 0.002, | |
| "reward": 0.30424928665161133, | |
| "reward_std": 0.23502430319786072, | |
| "rewards/format_reward": 0.08681320399045944, | |
| "rewards/judgement_reward": 0.1996888816356659, | |
| "rewards/token_reward": 0.01774725317955017, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.5824203491211, | |
| "epoch": 0.024390243902439025, | |
| "grad_norm": 1.931358237356469, | |
| "kl": 0.000339508056640625, | |
| "learning_rate": 2.5e-07, | |
| "loss": -0.0284, | |
| "reward": 0.27388590574264526, | |
| "reward_std": 0.22234365344047546, | |
| "rewards/format_reward": 0.09450550377368927, | |
| "rewards/judgement_reward": 0.1600947231054306, | |
| "rewards/token_reward": 0.019285714253783226, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.84066009521484, | |
| "epoch": 0.02926829268292683, | |
| "grad_norm": 2.102547432960352, | |
| "kl": 0.000278472900390625, | |
| "learning_rate": 3e-07, | |
| "loss": 0.0118, | |
| "reward": 0.5828344821929932, | |
| "reward_std": 0.27698564529418945, | |
| "rewards/format_reward": 0.11098900437355042, | |
| "rewards/judgement_reward": 0.4510761499404907, | |
| "rewards/token_reward": 0.020769229158759117, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.5769271850586, | |
| "epoch": 0.03414634146341464, | |
| "grad_norm": 2.187265324428969, | |
| "kl": 0.00026702880859375, | |
| "learning_rate": 3.5e-07, | |
| "loss": -0.0371, | |
| "reward": 0.38959935307502747, | |
| "reward_std": 0.274240106344223, | |
| "rewards/format_reward": 0.10659340023994446, | |
| "rewards/judgement_reward": 0.25861039757728577, | |
| "rewards/token_reward": 0.024395601823925972, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.87362670898438, | |
| "epoch": 0.03902439024390244, | |
| "grad_norm": 1.7011251355501271, | |
| "kl": 0.0003719329833984375, | |
| "learning_rate": 4e-07, | |
| "loss": 0.0269, | |
| "reward": 0.26069721579551697, | |
| "reward_std": 0.22703438997268677, | |
| "rewards/format_reward": 0.0912087932229042, | |
| "rewards/judgement_reward": 0.15185105800628662, | |
| "rewards/token_reward": 0.017637362703680992, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.25274658203125, | |
| "epoch": 0.04390243902439024, | |
| "grad_norm": 2.0455198863312387, | |
| "kl": 0.000316619873046875, | |
| "learning_rate": 4.5e-07, | |
| "loss": -0.0821, | |
| "reward": 0.34196677803993225, | |
| "reward_std": 0.25876346230506897, | |
| "rewards/format_reward": 0.09670329838991165, | |
| "rewards/judgement_reward": 0.22526347637176514, | |
| "rewards/token_reward": 0.019999999552965164, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.80769348144531, | |
| "epoch": 0.04878048780487805, | |
| "grad_norm": 1.8007886440567025, | |
| "kl": 0.000514984130859375, | |
| "learning_rate": 5e-07, | |
| "loss": -0.0259, | |
| "reward": 0.34238940477371216, | |
| "reward_std": 0.2460326850414276, | |
| "rewards/format_reward": 0.09560439735651016, | |
| "rewards/judgement_reward": 0.23140040040016174, | |
| "rewards/token_reward": 0.015384615398943424, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.19780731201172, | |
| "epoch": 0.05365853658536585, | |
| "grad_norm": 1.9673405461994626, | |
| "kl": 0.000698089599609375, | |
| "learning_rate": 5.5e-07, | |
| "loss": -0.0162, | |
| "reward": 0.18868696689605713, | |
| "reward_std": 0.15467074513435364, | |
| "rewards/format_reward": 0.07252748310565948, | |
| "rewards/judgement_reward": 0.10967598855495453, | |
| "rewards/token_reward": 0.006483516190201044, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.06593322753906, | |
| "epoch": 0.05853658536585366, | |
| "grad_norm": 1.9439922450073313, | |
| "kl": 0.000568389892578125, | |
| "learning_rate": 6e-07, | |
| "loss": -0.0065, | |
| "reward": 0.23565447330474854, | |
| "reward_std": 0.1971663385629654, | |
| "rewards/format_reward": 0.0912088081240654, | |
| "rewards/judgement_reward": 0.1310391128063202, | |
| "rewards/token_reward": 0.013406592421233654, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.83516693115234, | |
| "epoch": 0.06341463414634146, | |
| "grad_norm": 1.7492995284129258, | |
| "kl": 0.00131988525390625, | |
| "learning_rate": 6.5e-07, | |
| "loss": -0.0114, | |
| "reward": 0.4377107322216034, | |
| "reward_std": 0.2658415734767914, | |
| "rewards/format_reward": 0.09670329838991165, | |
| "rewards/judgement_reward": 0.320293128490448, | |
| "rewards/token_reward": 0.02071428671479225, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.25274658203125, | |
| "epoch": 0.06829268292682927, | |
| "grad_norm": 1.8852756312045653, | |
| "kl": 0.0023956298828125, | |
| "learning_rate": 7e-07, | |
| "loss": -0.1104, | |
| "reward": 0.25799688696861267, | |
| "reward_std": 0.22852738201618195, | |
| "rewards/format_reward": 0.08461539447307587, | |
| "rewards/judgement_reward": 0.1457441747188568, | |
| "rewards/token_reward": 0.027637362480163574, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.5934066772461, | |
| "epoch": 0.07317073170731707, | |
| "grad_norm": 1.8265395690821953, | |
| "kl": 0.003021240234375, | |
| "learning_rate": 7.5e-07, | |
| "loss": 0.0091, | |
| "reward": 0.3250506818294525, | |
| "reward_std": 0.2030942291021347, | |
| "rewards/format_reward": 0.08901099860668182, | |
| "rewards/judgement_reward": 0.2235121876001358, | |
| "rewards/token_reward": 0.012527472339570522, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.31318664550781, | |
| "epoch": 0.07804878048780488, | |
| "grad_norm": 1.8379653827038929, | |
| "kl": 0.0087890625, | |
| "learning_rate": 8e-07, | |
| "loss": -0.0047, | |
| "reward": 0.3267355263233185, | |
| "reward_std": 0.24120216071605682, | |
| "rewards/format_reward": 0.10109890252351761, | |
| "rewards/judgement_reward": 0.2026696354150772, | |
| "rewards/token_reward": 0.022967034950852394, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.23077392578125, | |
| "epoch": 0.08292682926829269, | |
| "grad_norm": 2.0529600972186666, | |
| "kl": 0.005584716796875, | |
| "learning_rate": 8.499999999999999e-07, | |
| "loss": 0.043, | |
| "reward": 0.3882504105567932, | |
| "reward_std": 0.24895574152469635, | |
| "rewards/format_reward": 0.1021978035569191, | |
| "rewards/judgement_reward": 0.2652834951877594, | |
| "rewards/token_reward": 0.020769229158759117, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.64835357666016, | |
| "epoch": 0.08780487804878048, | |
| "grad_norm": 2.1163090543204053, | |
| "kl": 0.01458740234375, | |
| "learning_rate": 9e-07, | |
| "loss": 0.04, | |
| "reward": 0.36192721128463745, | |
| "reward_std": 0.21096283197402954, | |
| "rewards/format_reward": 0.10659340769052505, | |
| "rewards/judgement_reward": 0.24775134027004242, | |
| "rewards/token_reward": 0.007582417689263821, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.26374053955078, | |
| "epoch": 0.09268292682926829, | |
| "grad_norm": 1.8165905290949755, | |
| "kl": 0.006988525390625, | |
| "learning_rate": 9.499999999999999e-07, | |
| "loss": 0.081, | |
| "reward": 0.3227555453777313, | |
| "reward_std": 0.20002064108848572, | |
| "rewards/format_reward": 0.10989010334014893, | |
| "rewards/judgement_reward": 0.20275558531284332, | |
| "rewards/token_reward": 0.01010989025235176, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.64835357666016, | |
| "epoch": 0.0975609756097561, | |
| "grad_norm": 1.967123469564035, | |
| "kl": 0.0120849609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0601, | |
| "reward": 0.474505752325058, | |
| "reward_std": 0.2401813268661499, | |
| "rewards/format_reward": 0.11978019773960114, | |
| "rewards/judgement_reward": 0.33972567319869995, | |
| "rewards/token_reward": 0.015000000596046448, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.8956069946289, | |
| "epoch": 0.1024390243902439, | |
| "grad_norm": 1.7567747825280924, | |
| "kl": 0.0108642578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0215, | |
| "reward": 0.27094441652297974, | |
| "reward_std": 0.20202507078647614, | |
| "rewards/format_reward": 0.10549449920654297, | |
| "rewards/judgement_reward": 0.15775761008262634, | |
| "rewards/token_reward": 0.007692308630794287, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.54945373535156, | |
| "epoch": 0.1073170731707317, | |
| "grad_norm": 1.8581558141105476, | |
| "kl": 0.08349609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.048, | |
| "reward": 0.4122963845729828, | |
| "reward_std": 0.24994409084320068, | |
| "rewards/format_reward": 0.1230769008398056, | |
| "rewards/judgement_reward": 0.27993375062942505, | |
| "rewards/token_reward": 0.009285714477300644, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 67.73626708984375, | |
| "epoch": 0.11219512195121951, | |
| "grad_norm": 1.8992395015405976, | |
| "kl": 0.01470947265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.014, | |
| "reward": 0.22567632794380188, | |
| "reward_std": 0.1741848886013031, | |
| "rewards/format_reward": 0.09560439735651016, | |
| "rewards/judgement_reward": 0.11776423454284668, | |
| "rewards/token_reward": 0.012307691387832165, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.0054931640625, | |
| "epoch": 0.11707317073170732, | |
| "grad_norm": 2.333890318615974, | |
| "kl": 0.01806640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0425, | |
| "reward": 0.3388529121875763, | |
| "reward_std": 0.22322086989879608, | |
| "rewards/format_reward": 0.12087910622358322, | |
| "rewards/judgement_reward": 0.20572103559970856, | |
| "rewards/token_reward": 0.012252748012542725, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.61538696289062, | |
| "epoch": 0.12195121951219512, | |
| "grad_norm": 2.5484341340291334, | |
| "kl": 0.021728515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0759, | |
| "reward": 0.6952180862426758, | |
| "reward_std": 0.2809818983078003, | |
| "rewards/format_reward": 0.15054939687252045, | |
| "rewards/judgement_reward": 0.5331300497055054, | |
| "rewards/token_reward": 0.011538460850715637, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.8956069946289, | |
| "epoch": 0.12682926829268293, | |
| "grad_norm": 2.00022024625464, | |
| "kl": 0.0308837890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0145, | |
| "reward": 0.4654199182987213, | |
| "reward_std": 0.24022048711776733, | |
| "rewards/format_reward": 0.13076920807361603, | |
| "rewards/judgement_reward": 0.3248704969882965, | |
| "rewards/token_reward": 0.0097802197560668, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 65.3846206665039, | |
| "epoch": 0.13170731707317074, | |
| "grad_norm": 2.133845945151958, | |
| "kl": 0.028564453125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0379, | |
| "reward": 0.6326283812522888, | |
| "reward_std": 0.25742557644844055, | |
| "rewards/format_reward": 0.14945051074028015, | |
| "rewards/judgement_reward": 0.4757600724697113, | |
| "rewards/token_reward": 0.0074175819754600525, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 66.95604705810547, | |
| "epoch": 0.13658536585365855, | |
| "grad_norm": 2.1425979650194793, | |
| "kl": 0.021484375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0393, | |
| "reward": 0.42065250873565674, | |
| "reward_std": 0.17824424803256989, | |
| "rewards/format_reward": 0.1131868064403534, | |
| "rewards/judgement_reward": 0.30175137519836426, | |
| "rewards/token_reward": 0.0057142856530845165, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.22527313232422, | |
| "epoch": 0.14146341463414633, | |
| "grad_norm": 2.187328509574426, | |
| "kl": 0.0264892578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0145, | |
| "reward": 0.5736148357391357, | |
| "reward_std": 0.250982403755188, | |
| "rewards/format_reward": 0.14175820350646973, | |
| "rewards/judgement_reward": 0.4122960567474365, | |
| "rewards/token_reward": 0.01956043764948845, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.989013671875, | |
| "epoch": 0.14634146341463414, | |
| "grad_norm": 2.0631362920367984, | |
| "kl": 0.0223388671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0436, | |
| "reward": 0.8461222648620605, | |
| "reward_std": 0.25097036361694336, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.6611222624778748, | |
| "rewards/token_reward": 0.023461539298295975, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 62.67582702636719, | |
| "epoch": 0.15121951219512195, | |
| "grad_norm": 2.5463613589005516, | |
| "kl": 0.037353515625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0348, | |
| "reward": 0.6614670753479004, | |
| "reward_std": 0.2477238029241562, | |
| "rewards/format_reward": 0.1439560055732727, | |
| "rewards/judgement_reward": 0.507401168346405, | |
| "rewards/token_reward": 0.01010989025235176, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 68.18681335449219, | |
| "epoch": 0.15609756097560976, | |
| "grad_norm": 2.157033445164365, | |
| "kl": 0.0264892578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0477, | |
| "reward": 0.6428108811378479, | |
| "reward_std": 0.21757948398590088, | |
| "rewards/format_reward": 0.14285710453987122, | |
| "rewards/judgement_reward": 0.4923712909221649, | |
| "rewards/token_reward": 0.007582417223602533, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 70.52747344970703, | |
| "epoch": 0.16097560975609757, | |
| "grad_norm": 2.5147644681680013, | |
| "kl": 1.71875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0331, | |
| "reward": 0.8551515340805054, | |
| "reward_std": 0.20301613211631775, | |
| "rewards/format_reward": 0.15934060513973236, | |
| "rewards/judgement_reward": 0.6783931255340576, | |
| "rewards/token_reward": 0.017417581751942635, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.4120864868164, | |
| "epoch": 0.16585365853658537, | |
| "grad_norm": 2.1819457001917804, | |
| "kl": 0.71484375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0018, | |
| "reward": 0.48153403401374817, | |
| "reward_std": 0.24979981780052185, | |
| "rewards/format_reward": 0.12417580932378769, | |
| "rewards/judgement_reward": 0.34016045928001404, | |
| "rewards/token_reward": 0.017197802662849426, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.0824203491211, | |
| "epoch": 0.17073170731707318, | |
| "grad_norm": 2.0349402588988084, | |
| "kl": 0.023193359375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0397, | |
| "reward": 0.49935343861579895, | |
| "reward_std": 0.25207024812698364, | |
| "rewards/format_reward": 0.1340659111738205, | |
| "rewards/judgement_reward": 0.3458918631076813, | |
| "rewards/token_reward": 0.019395604729652405, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.0769271850586, | |
| "epoch": 0.17560975609756097, | |
| "grad_norm": 2.204221432744273, | |
| "kl": 0.035400390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0172, | |
| "reward": 0.5841602087020874, | |
| "reward_std": 0.2741175889968872, | |
| "rewards/format_reward": 0.1450549066066742, | |
| "rewards/judgement_reward": 0.4198743999004364, | |
| "rewards/token_reward": 0.019230768084526062, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.20879364013672, | |
| "epoch": 0.18048780487804877, | |
| "grad_norm": 2.021038070522573, | |
| "kl": 0.038330078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0224, | |
| "reward": 0.41530290246009827, | |
| "reward_std": 0.2347797006368637, | |
| "rewards/format_reward": 0.11758241057395935, | |
| "rewards/judgement_reward": 0.2791491150856018, | |
| "rewards/token_reward": 0.01857142709195614, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.3846206665039, | |
| "epoch": 0.18536585365853658, | |
| "grad_norm": 2.2222191685212156, | |
| "kl": 0.044189453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0314, | |
| "reward": 0.6318976879119873, | |
| "reward_std": 0.26240062713623047, | |
| "rewards/format_reward": 0.1351647973060608, | |
| "rewards/judgement_reward": 0.4639304578304291, | |
| "rewards/token_reward": 0.032802194356918335, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.52747344970703, | |
| "epoch": 0.1902439024390244, | |
| "grad_norm": 1.8969688577935941, | |
| "kl": 0.138671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0619, | |
| "reward": 0.7222000360488892, | |
| "reward_std": 0.268084853887558, | |
| "rewards/format_reward": 0.15384609997272491, | |
| "rewards/judgement_reward": 0.5535735487937927, | |
| "rewards/token_reward": 0.014780220575630665, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.75274658203125, | |
| "epoch": 0.1951219512195122, | |
| "grad_norm": 2.1456284767626763, | |
| "kl": 0.046142578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0734, | |
| "reward": 0.8482385277748108, | |
| "reward_std": 0.24352890253067017, | |
| "rewards/format_reward": 0.1560439020395279, | |
| "rewards/judgement_reward": 0.6564801335334778, | |
| "rewards/token_reward": 0.0357142835855484, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.15933990478516, | |
| "epoch": 0.2, | |
| "grad_norm": 2.09640812853542, | |
| "kl": 0.16796875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0156, | |
| "reward": 0.8189014792442322, | |
| "reward_std": 0.2852559983730316, | |
| "rewards/format_reward": 0.15824170410633087, | |
| "rewards/judgement_reward": 0.6289563775062561, | |
| "rewards/token_reward": 0.031703293323516846, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.30769348144531, | |
| "epoch": 0.2048780487804878, | |
| "grad_norm": 2.05137630399559, | |
| "kl": 0.030029296875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0138, | |
| "reward": 0.9518312215805054, | |
| "reward_std": 0.21282121539115906, | |
| "rewards/format_reward": 0.16923069953918457, | |
| "rewards/judgement_reward": 0.759743332862854, | |
| "rewards/token_reward": 0.022857142612338066, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.30220031738281, | |
| "epoch": 0.2097560975609756, | |
| "grad_norm": 2.212876795222977, | |
| "kl": 0.03857421875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.042, | |
| "reward": 0.5973189473152161, | |
| "reward_std": 0.2632991671562195, | |
| "rewards/format_reward": 0.13186810910701752, | |
| "rewards/judgement_reward": 0.4431980848312378, | |
| "rewards/token_reward": 0.022252749651670456, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.78022003173828, | |
| "epoch": 0.2146341463414634, | |
| "grad_norm": 1.909917279230211, | |
| "kl": 0.02197265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0097, | |
| "reward": 0.6979678869247437, | |
| "reward_std": 0.2731499671936035, | |
| "rewards/format_reward": 0.1461537927389145, | |
| "rewards/judgement_reward": 0.5200557112693787, | |
| "rewards/token_reward": 0.03175824135541916, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.13736724853516, | |
| "epoch": 0.21951219512195122, | |
| "grad_norm": 2.00637254655722, | |
| "kl": 0.031982421875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0395, | |
| "reward": 0.5393092036247253, | |
| "reward_std": 0.18171927332878113, | |
| "rewards/format_reward": 0.12197799980640411, | |
| "rewards/judgement_reward": 0.40848490595817566, | |
| "rewards/token_reward": 0.008846154436469078, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.3901138305664, | |
| "epoch": 0.22439024390243903, | |
| "grad_norm": 3.1756908083902964, | |
| "kl": 0.032958984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0354, | |
| "reward": 0.5674500465393066, | |
| "reward_std": 0.21071720123291016, | |
| "rewards/format_reward": 0.13186810910701752, | |
| "rewards/judgement_reward": 0.4111863970756531, | |
| "rewards/token_reward": 0.024395601823925972, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.52747344970703, | |
| "epoch": 0.22926829268292684, | |
| "grad_norm": 2.006041173831088, | |
| "kl": 0.0302734375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0108, | |
| "reward": 0.5255321860313416, | |
| "reward_std": 0.24098682403564453, | |
| "rewards/format_reward": 0.12747250497341156, | |
| "rewards/judgement_reward": 0.3749277889728546, | |
| "rewards/token_reward": 0.02313186600804329, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.30220031738281, | |
| "epoch": 0.23414634146341465, | |
| "grad_norm": 2.080512783491811, | |
| "kl": 0.0257568359375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0174, | |
| "reward": 0.6429842710494995, | |
| "reward_std": 0.2190992385149002, | |
| "rewards/format_reward": 0.14615380764007568, | |
| "rewards/judgement_reward": 0.47715994715690613, | |
| "rewards/token_reward": 0.019670329988002777, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.19780731201172, | |
| "epoch": 0.23902439024390243, | |
| "grad_norm": 2.0510274772907042, | |
| "kl": 0.0220947265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0791, | |
| "reward": 0.8461357355117798, | |
| "reward_std": 0.2294541895389557, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.6526740789413452, | |
| "rewards/token_reward": 0.028626371175050735, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.05494689941406, | |
| "epoch": 0.24390243902439024, | |
| "grad_norm": 2.1607651542772115, | |
| "kl": 0.0244140625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0118, | |
| "reward": 0.7917323708534241, | |
| "reward_std": 0.2613984942436218, | |
| "rewards/format_reward": 0.15714280307292938, | |
| "rewards/judgement_reward": 0.6060182452201843, | |
| "rewards/token_reward": 0.02857142873108387, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.94505310058594, | |
| "epoch": 0.24878048780487805, | |
| "grad_norm": 2.4101187996457467, | |
| "kl": 0.06787109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0128, | |
| "reward": 0.6344149112701416, | |
| "reward_std": 0.2056046426296234, | |
| "rewards/format_reward": 0.1340659111738205, | |
| "rewards/judgement_reward": 0.4821070730686188, | |
| "rewards/token_reward": 0.01824175752699375, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.18132019042969, | |
| "epoch": 0.25365853658536586, | |
| "grad_norm": 2.0276407359997446, | |
| "kl": 0.205078125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0588, | |
| "reward": 0.9314945340156555, | |
| "reward_std": 0.17086516320705414, | |
| "rewards/format_reward": 0.1659339964389801, | |
| "rewards/judgement_reward": 0.7470438480377197, | |
| "rewards/token_reward": 0.018516482785344124, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.30769348144531, | |
| "epoch": 0.25853658536585367, | |
| "grad_norm": 2.17785136004001, | |
| "kl": 0.031982421875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0705, | |
| "reward": 0.7751470804214478, | |
| "reward_std": 0.2271140068769455, | |
| "rewards/format_reward": 0.1549450010061264, | |
| "rewards/judgement_reward": 0.5904769897460938, | |
| "rewards/token_reward": 0.029725274071097374, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.989013671875, | |
| "epoch": 0.2634146341463415, | |
| "grad_norm": 2.6022497270890868, | |
| "kl": 0.0400390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0112, | |
| "reward": 0.7827267646789551, | |
| "reward_std": 0.2629249691963196, | |
| "rewards/format_reward": 0.1549450010061264, | |
| "rewards/judgement_reward": 0.6101441979408264, | |
| "rewards/token_reward": 0.017637362703680992, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.96703338623047, | |
| "epoch": 0.2682926829268293, | |
| "grad_norm": 1.9363620619923776, | |
| "kl": 0.032958984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0014, | |
| "reward": 0.579794704914093, | |
| "reward_std": 0.26253968477249146, | |
| "rewards/format_reward": 0.13846150040626526, | |
| "rewards/judgement_reward": 0.41259682178497314, | |
| "rewards/token_reward": 0.028736261650919914, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.9945068359375, | |
| "epoch": 0.2731707317073171, | |
| "grad_norm": 2.2800506149734696, | |
| "kl": 0.06005859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.035, | |
| "reward": 0.7825473546981812, | |
| "reward_std": 0.2477075755596161, | |
| "rewards/format_reward": 0.15934060513973236, | |
| "rewards/judgement_reward": 0.6038658618927002, | |
| "rewards/token_reward": 0.01934065856039524, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.19230651855469, | |
| "epoch": 0.2780487804878049, | |
| "grad_norm": 1.9316889087097737, | |
| "kl": 0.03369140625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0151, | |
| "reward": 0.7461085319519043, | |
| "reward_std": 0.22608627378940582, | |
| "rewards/format_reward": 0.14835159480571747, | |
| "rewards/judgement_reward": 0.5816579461097717, | |
| "rewards/token_reward": 0.016098899766802788, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.45604705810547, | |
| "epoch": 0.28292682926829266, | |
| "grad_norm": 2.891000369424163, | |
| "kl": 0.09228515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0333, | |
| "reward": 0.7758737206459045, | |
| "reward_std": 0.27175047993659973, | |
| "rewards/format_reward": 0.15714281797409058, | |
| "rewards/judgement_reward": 0.5963131785392761, | |
| "rewards/token_reward": 0.02241758443415165, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.02198028564453, | |
| "epoch": 0.28780487804878047, | |
| "grad_norm": 2.08130255800055, | |
| "kl": 0.0277099609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0011, | |
| "reward": 0.7745002508163452, | |
| "reward_std": 0.24488268792629242, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.5817528367042542, | |
| "rewards/token_reward": 0.031208788976073265, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.19230651855469, | |
| "epoch": 0.2926829268292683, | |
| "grad_norm": 2.325364169755121, | |
| "kl": 0.048583984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.054, | |
| "reward": 0.7218204736709595, | |
| "reward_std": 0.22538162767887115, | |
| "rewards/format_reward": 0.1439560055732727, | |
| "rewards/judgement_reward": 0.5538533926010132, | |
| "rewards/token_reward": 0.02401098981499672, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.14835357666016, | |
| "epoch": 0.2975609756097561, | |
| "grad_norm": 2.199589014022037, | |
| "kl": 0.0289306640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0617, | |
| "reward": 0.6991139054298401, | |
| "reward_std": 0.22941938042640686, | |
| "rewards/format_reward": 0.1450549066066742, | |
| "rewards/judgement_reward": 0.5255423784255981, | |
| "rewards/token_reward": 0.028516482561826706, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.95055389404297, | |
| "epoch": 0.3024390243902439, | |
| "grad_norm": 2.0508974708861576, | |
| "kl": 0.02587890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.04, | |
| "reward": 0.6912637948989868, | |
| "reward_std": 0.22075381875038147, | |
| "rewards/format_reward": 0.15274719893932343, | |
| "rewards/judgement_reward": 0.5231318473815918, | |
| "rewards/token_reward": 0.015384615398943424, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.80769348144531, | |
| "epoch": 0.3073170731707317, | |
| "grad_norm": 2.2205627029567467, | |
| "kl": 0.035888671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0696, | |
| "reward": 0.7073810696601868, | |
| "reward_std": 0.22409050166606903, | |
| "rewards/format_reward": 0.15714280307292938, | |
| "rewards/judgement_reward": 0.514194130897522, | |
| "rewards/token_reward": 0.036043956875801086, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 70.52747344970703, | |
| "epoch": 0.3121951219512195, | |
| "grad_norm": 2.0916293721018544, | |
| "kl": 0.032958984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0537, | |
| "reward": 0.998910129070282, | |
| "reward_std": 0.2128116488456726, | |
| "rewards/format_reward": 0.17032960057258606, | |
| "rewards/judgement_reward": 0.800338625907898, | |
| "rewards/token_reward": 0.028241755440831184, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.0934066772461, | |
| "epoch": 0.3170731707317073, | |
| "grad_norm": 3.671459315517818, | |
| "kl": 0.056640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0158, | |
| "reward": 0.8368141651153564, | |
| "reward_std": 0.24064061045646667, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.6522535681724548, | |
| "rewards/token_reward": 0.02302197553217411, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.41758728027344, | |
| "epoch": 0.32195121951219513, | |
| "grad_norm": 2.0126846364586273, | |
| "kl": 0.026123046875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0154, | |
| "reward": 0.809525728225708, | |
| "reward_std": 0.2356753945350647, | |
| "rewards/format_reward": 0.16373620927333832, | |
| "rewards/judgement_reward": 0.6260641813278198, | |
| "rewards/token_reward": 0.019725274294614792, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.15933990478516, | |
| "epoch": 0.32682926829268294, | |
| "grad_norm": 1.851476238680362, | |
| "kl": 0.0267333984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0033, | |
| "reward": 0.6307772397994995, | |
| "reward_std": 0.23441998660564423, | |
| "rewards/format_reward": 0.14835159480571747, | |
| "rewards/judgement_reward": 0.45868924260139465, | |
| "rewards/token_reward": 0.023736262694001198, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.77472686767578, | |
| "epoch": 0.33170731707317075, | |
| "grad_norm": 1.9067629127088346, | |
| "kl": 0.027587890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0166, | |
| "reward": 0.6367327570915222, | |
| "reward_std": 0.207670658826828, | |
| "rewards/format_reward": 0.15054941177368164, | |
| "rewards/judgement_reward": 0.46409520506858826, | |
| "rewards/token_reward": 0.022087913006544113, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.63186645507812, | |
| "epoch": 0.33658536585365856, | |
| "grad_norm": 2.192665819859577, | |
| "kl": 0.058349609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0979, | |
| "reward": 0.8123928904533386, | |
| "reward_std": 0.23223978281021118, | |
| "rewards/format_reward": 0.16263730823993683, | |
| "rewards/judgement_reward": 0.6219531297683716, | |
| "rewards/token_reward": 0.027802197262644768, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.26374053955078, | |
| "epoch": 0.34146341463414637, | |
| "grad_norm": 2.339003134381284, | |
| "kl": 0.02392578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0611, | |
| "reward": 0.9417235255241394, | |
| "reward_std": 0.18349522352218628, | |
| "rewards/format_reward": 0.16813181340694427, | |
| "rewards/judgement_reward": 0.7577126622200012, | |
| "rewards/token_reward": 0.01587912067770958, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.05494689941406, | |
| "epoch": 0.3463414634146341, | |
| "grad_norm": 2.3107542796374916, | |
| "kl": 0.0244140625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0253, | |
| "reward": 0.8421667218208313, | |
| "reward_std": 0.22592146694660187, | |
| "rewards/format_reward": 0.15384609997272491, | |
| "rewards/judgement_reward": 0.6690347790718079, | |
| "rewards/token_reward": 0.019285714253783226, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.9120864868164, | |
| "epoch": 0.35121951219512193, | |
| "grad_norm": 1.8500979548941974, | |
| "kl": 0.0191650390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0408, | |
| "reward": 0.8825385570526123, | |
| "reward_std": 0.23543551564216614, | |
| "rewards/format_reward": 0.17362630367279053, | |
| "rewards/judgement_reward": 0.6919341087341309, | |
| "rewards/token_reward": 0.01697802171111107, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.62088012695312, | |
| "epoch": 0.35609756097560974, | |
| "grad_norm": 1.7964524052061408, | |
| "kl": 0.0262451171875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0512, | |
| "reward": 0.7030321359634399, | |
| "reward_std": 0.20544546842575073, | |
| "rewards/format_reward": 0.14615380764007568, | |
| "rewards/judgement_reward": 0.5344606637954712, | |
| "rewards/token_reward": 0.0224175825715065, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.87911987304688, | |
| "epoch": 0.36097560975609755, | |
| "grad_norm": 1.9935947664757243, | |
| "kl": 0.19921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.055, | |
| "reward": 0.9933971762657166, | |
| "reward_std": 0.22045965492725372, | |
| "rewards/format_reward": 0.17252740263938904, | |
| "rewards/judgement_reward": 0.7981772422790527, | |
| "rewards/token_reward": 0.022692309692502022, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.4011001586914, | |
| "epoch": 0.36585365853658536, | |
| "grad_norm": 2.001748403246846, | |
| "kl": 0.0308837890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0664, | |
| "reward": 0.6825388073921204, | |
| "reward_std": 0.2821784019470215, | |
| "rewards/format_reward": 0.1560439020395279, | |
| "rewards/judgement_reward": 0.49479159712791443, | |
| "rewards/token_reward": 0.031703293323516846, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.41758728027344, | |
| "epoch": 0.37073170731707317, | |
| "grad_norm": 2.2248051545634824, | |
| "kl": 0.0302734375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0628, | |
| "reward": 0.7976081371307373, | |
| "reward_std": 0.24464865028858185, | |
| "rewards/format_reward": 0.16263730823993683, | |
| "rewards/judgement_reward": 0.611564040184021, | |
| "rewards/token_reward": 0.02340659312903881, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.8901138305664, | |
| "epoch": 0.375609756097561, | |
| "grad_norm": 2.0499939459920835, | |
| "kl": 0.03662109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0634, | |
| "reward": 0.6835858821868896, | |
| "reward_std": 0.21102023124694824, | |
| "rewards/format_reward": 0.1351647973060608, | |
| "rewards/judgement_reward": 0.5263329744338989, | |
| "rewards/token_reward": 0.022087913006544113, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.62637329101562, | |
| "epoch": 0.3804878048780488, | |
| "grad_norm": 2.266766438478251, | |
| "kl": 0.02880859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0109, | |
| "reward": 0.8073420524597168, | |
| "reward_std": 0.23677751421928406, | |
| "rewards/format_reward": 0.1560439020395279, | |
| "rewards/judgement_reward": 0.6212978959083557, | |
| "rewards/token_reward": 0.029999997466802597, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.93955993652344, | |
| "epoch": 0.3853658536585366, | |
| "grad_norm": 2.4759945771013867, | |
| "kl": 0.06640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0217, | |
| "reward": 0.6546903252601624, | |
| "reward_std": 0.26877561211586, | |
| "rewards/format_reward": 0.15164829790592194, | |
| "rewards/judgement_reward": 0.4823826551437378, | |
| "rewards/token_reward": 0.020659340545535088, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.65384674072266, | |
| "epoch": 0.3902439024390244, | |
| "grad_norm": 1.945518643660678, | |
| "kl": 0.047119140625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0002, | |
| "reward": 0.7612788081169128, | |
| "reward_std": 0.1894712746143341, | |
| "rewards/format_reward": 0.13736259937286377, | |
| "rewards/judgement_reward": 0.6034765243530273, | |
| "rewards/token_reward": 0.02043955959379673, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.78571319580078, | |
| "epoch": 0.3951219512195122, | |
| "grad_norm": 2.0901126990153127, | |
| "kl": 0.11474609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0731, | |
| "reward": 1.0548338890075684, | |
| "reward_std": 0.21029122173786163, | |
| "rewards/format_reward": 0.1769230216741562, | |
| "rewards/judgement_reward": 0.8410976529121399, | |
| "rewards/token_reward": 0.03681318834424019, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.8956069946289, | |
| "epoch": 0.4, | |
| "grad_norm": 1.7461151621773074, | |
| "kl": 0.03271484375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0471, | |
| "reward": 0.7646914720535278, | |
| "reward_std": 0.2543509900569916, | |
| "rewards/format_reward": 0.15714281797409058, | |
| "rewards/judgement_reward": 0.5825485587120056, | |
| "rewards/token_reward": 0.02499999850988388, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.9065933227539, | |
| "epoch": 0.40487804878048783, | |
| "grad_norm": 2.0106379862864663, | |
| "kl": 0.0311279296875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0125, | |
| "reward": 0.7661330699920654, | |
| "reward_std": 0.2671906650066376, | |
| "rewards/format_reward": 0.15934060513973236, | |
| "rewards/judgement_reward": 0.5814077854156494, | |
| "rewards/token_reward": 0.02538461610674858, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.64835357666016, | |
| "epoch": 0.4097560975609756, | |
| "grad_norm": 1.9108709969888935, | |
| "kl": 0.283203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0065, | |
| "reward": 0.8318064212799072, | |
| "reward_std": 0.23601563274860382, | |
| "rewards/format_reward": 0.15934060513973236, | |
| "rewards/judgement_reward": 0.6468064785003662, | |
| "rewards/token_reward": 0.025659339502453804, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.02747344970703, | |
| "epoch": 0.4146341463414634, | |
| "grad_norm": 1.890024675225077, | |
| "kl": 0.1630859375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0754, | |
| "reward": 0.9830853343009949, | |
| "reward_std": 0.22543151676654816, | |
| "rewards/format_reward": 0.17362630367279053, | |
| "rewards/judgement_reward": 0.7781403064727783, | |
| "rewards/token_reward": 0.031318679451942444, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.0824203491211, | |
| "epoch": 0.4195121951219512, | |
| "grad_norm": 2.47037218372719, | |
| "kl": 0.044921875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0005, | |
| "reward": 0.6921790242195129, | |
| "reward_std": 0.26250430941581726, | |
| "rewards/format_reward": 0.14725270867347717, | |
| "rewards/judgement_reward": 0.5239920616149902, | |
| "rewards/token_reward": 0.02093406394124031, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.28571319580078, | |
| "epoch": 0.424390243902439, | |
| "grad_norm": 2.079365060925443, | |
| "kl": 0.0267333984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0605, | |
| "reward": 0.7566412687301636, | |
| "reward_std": 0.21086536347866058, | |
| "rewards/format_reward": 0.15824170410633087, | |
| "rewards/judgement_reward": 0.5732895731925964, | |
| "rewards/token_reward": 0.02510989084839821, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.31867980957031, | |
| "epoch": 0.4292682926829268, | |
| "grad_norm": 2.1529982274280863, | |
| "kl": 0.0255126953125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0807, | |
| "reward": 0.8711549639701843, | |
| "reward_std": 0.19136983156204224, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.6887921690940857, | |
| "rewards/token_reward": 0.02082417532801628, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.85164642333984, | |
| "epoch": 0.43414634146341463, | |
| "grad_norm": 1.9476865855709369, | |
| "kl": 0.03466796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0521, | |
| "reward": 0.849588930606842, | |
| "reward_std": 0.22347387671470642, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.6607428193092346, | |
| "rewards/token_reward": 0.02401098981499672, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.53296661376953, | |
| "epoch": 0.43902439024390244, | |
| "grad_norm": 1.9988855691892198, | |
| "kl": 0.3203125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0217, | |
| "reward": 0.794158935546875, | |
| "reward_std": 0.2382083535194397, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.6096535921096802, | |
| "rewards/token_reward": 0.022967033088207245, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.9065933227539, | |
| "epoch": 0.44390243902439025, | |
| "grad_norm": 2.3812234433330333, | |
| "kl": 0.039794921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.066, | |
| "reward": 0.9384704828262329, | |
| "reward_std": 0.19025777280330658, | |
| "rewards/format_reward": 0.1659340113401413, | |
| "rewards/judgement_reward": 0.7457780838012695, | |
| "rewards/token_reward": 0.026758242398500443, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.04396057128906, | |
| "epoch": 0.44878048780487806, | |
| "grad_norm": 2.6148084459815206, | |
| "kl": 0.04345703125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0984, | |
| "reward": 1.0894677639007568, | |
| "reward_std": 0.1785333752632141, | |
| "rewards/format_reward": 0.18241752684116364, | |
| "rewards/judgement_reward": 0.880236804485321, | |
| "rewards/token_reward": 0.026813184842467308, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.24725341796875, | |
| "epoch": 0.45365853658536587, | |
| "grad_norm": 1.9266274957793252, | |
| "kl": 0.10205078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0333, | |
| "reward": 0.6261184215545654, | |
| "reward_std": 0.2962965965270996, | |
| "rewards/format_reward": 0.1450549066066742, | |
| "rewards/judgement_reward": 0.4455689489841461, | |
| "rewards/token_reward": 0.03549450263381004, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.0714340209961, | |
| "epoch": 0.4585365853658537, | |
| "grad_norm": 2.359476534290484, | |
| "kl": 0.0693359375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0619, | |
| "reward": 0.6673313975334167, | |
| "reward_std": 0.20072828233242035, | |
| "rewards/format_reward": 0.13736259937286377, | |
| "rewards/judgement_reward": 0.5074961185455322, | |
| "rewards/token_reward": 0.022472526878118515, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.46703338623047, | |
| "epoch": 0.4634146341463415, | |
| "grad_norm": 2.1867681379205126, | |
| "kl": 0.045166015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0384, | |
| "reward": 0.8676297664642334, | |
| "reward_std": 0.2376934140920639, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.6711461544036865, | |
| "rewards/token_reward": 0.03164835274219513, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.30769348144531, | |
| "epoch": 0.4682926829268293, | |
| "grad_norm": 1.7938305263175633, | |
| "kl": 0.0294189453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0293, | |
| "reward": 0.8810251355171204, | |
| "reward_std": 0.22088079154491425, | |
| "rewards/format_reward": 0.1659339964389801, | |
| "rewards/judgement_reward": 0.6851460337638855, | |
| "rewards/token_reward": 0.02994505502283573, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.96154022216797, | |
| "epoch": 0.47317073170731705, | |
| "grad_norm": 1.9835104874280258, | |
| "kl": 0.0262451171875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.031, | |
| "reward": 1.0301648378372192, | |
| "reward_std": 0.23829680681228638, | |
| "rewards/format_reward": 0.17912080883979797, | |
| "rewards/judgement_reward": 0.8086262941360474, | |
| "rewards/token_reward": 0.042417578399181366, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.10989379882812, | |
| "epoch": 0.47804878048780486, | |
| "grad_norm": 1.8068829512126077, | |
| "kl": 0.025390625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0435, | |
| "reward": 0.8396397233009338, | |
| "reward_std": 0.20062671601772308, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.6437055468559265, | |
| "rewards/token_reward": 0.031098900362849236, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.79670715332031, | |
| "epoch": 0.48292682926829267, | |
| "grad_norm": 1.8013193104560348, | |
| "kl": 0.0294189453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.021, | |
| "reward": 0.8207690119743347, | |
| "reward_std": 0.18635249137878418, | |
| "rewards/format_reward": 0.16263730823993683, | |
| "rewards/judgement_reward": 0.6401647329330444, | |
| "rewards/token_reward": 0.01796703413128853, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.29121398925781, | |
| "epoch": 0.4878048780487805, | |
| "grad_norm": 1.8541360087649421, | |
| "kl": 0.03173828125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0518, | |
| "reward": 0.8696525692939758, | |
| "reward_std": 0.2140418291091919, | |
| "rewards/format_reward": 0.16813179850578308, | |
| "rewards/judgement_reward": 0.6785534620285034, | |
| "rewards/token_reward": 0.022967033088207245, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.42857360839844, | |
| "epoch": 0.4926829268292683, | |
| "grad_norm": 2.3437263624638858, | |
| "kl": 0.03759765625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0011, | |
| "reward": 0.7790926694869995, | |
| "reward_std": 0.29135364294052124, | |
| "rewards/format_reward": 0.16373620927333832, | |
| "rewards/judgement_reward": 0.5860158205032349, | |
| "rewards/token_reward": 0.029340656474232674, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 70.17033386230469, | |
| "epoch": 0.4975609756097561, | |
| "grad_norm": 2.1875591465659765, | |
| "kl": 0.05029296875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0458, | |
| "reward": 0.9169327616691589, | |
| "reward_std": 0.21496839821338654, | |
| "rewards/format_reward": 0.17142850160598755, | |
| "rewards/judgement_reward": 0.7243505120277405, | |
| "rewards/token_reward": 0.021153846755623817, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.53845977783203, | |
| "epoch": 0.5024390243902439, | |
| "grad_norm": 2.2860621039378097, | |
| "kl": 0.052978515625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0511, | |
| "reward": 0.8492118120193481, | |
| "reward_std": 0.2524387836456299, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.6557502746582031, | |
| "rewards/token_reward": 0.028626374900341034, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.21977996826172, | |
| "epoch": 0.5073170731707317, | |
| "grad_norm": 1.9503004419157008, | |
| "kl": 0.03857421875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1021, | |
| "reward": 0.9117417335510254, | |
| "reward_std": 0.21893596649169922, | |
| "rewards/format_reward": 0.16813181340694427, | |
| "rewards/judgement_reward": 0.7197636961936951, | |
| "rewards/token_reward": 0.023846155032515526, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 69.92308044433594, | |
| "epoch": 0.5121951219512195, | |
| "grad_norm": 2.2102406225271696, | |
| "kl": 0.0732421875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0789, | |
| "reward": 0.9815363883972168, | |
| "reward_std": 0.1895749419927597, | |
| "rewards/format_reward": 0.17472520470619202, | |
| "rewards/judgement_reward": 0.7850527167320251, | |
| "rewards/token_reward": 0.021758243441581726, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.5, | |
| "epoch": 0.5170731707317073, | |
| "grad_norm": 1.8701663684695582, | |
| "kl": 0.029052734375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0526, | |
| "reward": 1.0305202007293701, | |
| "reward_std": 0.21711336076259613, | |
| "rewards/format_reward": 0.17802190780639648, | |
| "rewards/judgement_reward": 0.8127729296684265, | |
| "rewards/token_reward": 0.039725273847579956, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.36813354492188, | |
| "epoch": 0.5219512195121951, | |
| "grad_norm": 2.1504947624651005, | |
| "kl": 0.041015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0108, | |
| "reward": 0.8238873481750488, | |
| "reward_std": 0.28225311636924744, | |
| "rewards/format_reward": 0.1659340113401413, | |
| "rewards/judgement_reward": 0.6267994046211243, | |
| "rewards/token_reward": 0.0311538465321064, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.24176025390625, | |
| "epoch": 0.526829268292683, | |
| "grad_norm": 1.7794056509498575, | |
| "kl": 0.0311279296875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0823, | |
| "reward": 0.9624238014221191, | |
| "reward_std": 0.2303503304719925, | |
| "rewards/format_reward": 0.17252740263938904, | |
| "rewards/judgement_reward": 0.7584677934646606, | |
| "rewards/token_reward": 0.03142856806516647, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.4120864868164, | |
| "epoch": 0.5317073170731708, | |
| "grad_norm": 2.504687498054589, | |
| "kl": 0.05712890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.072, | |
| "reward": 0.976593554019928, | |
| "reward_std": 0.23477815091609955, | |
| "rewards/format_reward": 0.1758241206407547, | |
| "rewards/judgement_reward": 0.7797802090644836, | |
| "rewards/token_reward": 0.020989011973142624, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.32418060302734, | |
| "epoch": 0.5365853658536586, | |
| "grad_norm": 1.9342624784665794, | |
| "kl": 0.0419921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0906, | |
| "reward": 1.0066561698913574, | |
| "reward_std": 0.18863876163959503, | |
| "rewards/format_reward": 0.18131859600543976, | |
| "rewards/judgement_reward": 0.7958320379257202, | |
| "rewards/token_reward": 0.029505494982004166, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.02747344970703, | |
| "epoch": 0.5414634146341464, | |
| "grad_norm": 3.6005569321956736, | |
| "kl": 0.07275390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0868, | |
| "reward": 0.9240605235099792, | |
| "reward_std": 0.2554757595062256, | |
| "rewards/format_reward": 0.16813181340694427, | |
| "rewards/judgement_reward": 0.7308735251426697, | |
| "rewards/token_reward": 0.025054944679141045, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.05494689941406, | |
| "epoch": 0.5463414634146342, | |
| "grad_norm": 2.384945004137537, | |
| "kl": 0.0498046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0406, | |
| "reward": 0.934312105178833, | |
| "reward_std": 0.25367388129234314, | |
| "rewards/format_reward": 0.1758241057395935, | |
| "rewards/judgement_reward": 0.7368395924568176, | |
| "rewards/token_reward": 0.021648351103067398, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 81.47802734375, | |
| "epoch": 0.551219512195122, | |
| "grad_norm": 2.4494451662329446, | |
| "kl": 0.04150390625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0659, | |
| "reward": 0.9666280746459961, | |
| "reward_std": 0.26005613803863525, | |
| "rewards/format_reward": 0.176923006772995, | |
| "rewards/judgement_reward": 0.7433313131332397, | |
| "rewards/token_reward": 0.046373624354600906, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.86264038085938, | |
| "epoch": 0.5560975609756098, | |
| "grad_norm": 1.8974487896916954, | |
| "kl": 0.17578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0702, | |
| "reward": 1.1044301986694336, | |
| "reward_std": 0.20463985204696655, | |
| "rewards/format_reward": 0.18461531400680542, | |
| "rewards/judgement_reward": 0.8778916597366333, | |
| "rewards/token_reward": 0.04192307963967323, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 73.41758728027344, | |
| "epoch": 0.5609756097560976, | |
| "grad_norm": 2.0736124082447858, | |
| "kl": 0.055908203125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0178, | |
| "reward": 0.45087727904319763, | |
| "reward_std": 0.16333483159542084, | |
| "rewards/format_reward": 0.12087910622358322, | |
| "rewards/judgement_reward": 0.3127453625202179, | |
| "rewards/token_reward": 0.01725274696946144, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 72.31318664550781, | |
| "epoch": 0.5658536585365853, | |
| "grad_norm": 1.9880834210174607, | |
| "kl": 0.265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0914, | |
| "reward": 1.0456217527389526, | |
| "reward_std": 0.1933256834745407, | |
| "rewards/format_reward": 0.17912080883979797, | |
| "rewards/judgement_reward": 0.8432590365409851, | |
| "rewards/token_reward": 0.023241758346557617, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.79670715332031, | |
| "epoch": 0.5707317073170731, | |
| "grad_norm": 2.0598303213859634, | |
| "kl": 0.03173828125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0638, | |
| "reward": 1.015529751777649, | |
| "reward_std": 0.21438416838645935, | |
| "rewards/format_reward": 0.1769230216741562, | |
| "rewards/judgement_reward": 0.8019583821296692, | |
| "rewards/token_reward": 0.03664834797382355, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.15384674072266, | |
| "epoch": 0.5756097560975609, | |
| "grad_norm": 2.1302776398753327, | |
| "kl": 0.0322265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1029, | |
| "reward": 1.0583690404891968, | |
| "reward_std": 0.16479381918907166, | |
| "rewards/format_reward": 0.1857142299413681, | |
| "rewards/judgement_reward": 0.8432590365409851, | |
| "rewards/token_reward": 0.029395602643489838, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.35165405273438, | |
| "epoch": 0.5804878048780487, | |
| "grad_norm": 2.139306907057807, | |
| "kl": 0.040771484375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0908, | |
| "reward": 0.9103569984436035, | |
| "reward_std": 0.2012585699558258, | |
| "rewards/format_reward": 0.1659339964389801, | |
| "rewards/judgement_reward": 0.7116758227348328, | |
| "rewards/token_reward": 0.03274725377559662, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 81.20879364013672, | |
| "epoch": 0.5853658536585366, | |
| "grad_norm": 1.8649093478118766, | |
| "kl": 0.04931640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0399, | |
| "reward": 0.9918122887611389, | |
| "reward_std": 0.2688324749469757, | |
| "rewards/format_reward": 0.176923006772995, | |
| "rewards/judgement_reward": 0.7671419978141785, | |
| "rewards/token_reward": 0.04774724692106247, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.04396057128906, | |
| "epoch": 0.5902439024390244, | |
| "grad_norm": 2.0906208019011614, | |
| "kl": 0.05322265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.084, | |
| "reward": 0.9650415778160095, | |
| "reward_std": 0.21533620357513428, | |
| "rewards/format_reward": 0.17362630367279053, | |
| "rewards/judgement_reward": 0.7517996430397034, | |
| "rewards/token_reward": 0.03961538150906563, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.62088012695312, | |
| "epoch": 0.5951219512195122, | |
| "grad_norm": 2.6404896615388513, | |
| "kl": 0.039306640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0209, | |
| "reward": 0.8149644732475281, | |
| "reward_std": 0.21845516562461853, | |
| "rewards/format_reward": 0.15824170410633087, | |
| "rewards/judgement_reward": 0.6252389550209045, | |
| "rewards/token_reward": 0.03148351609706879, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.19230651855469, | |
| "epoch": 0.6, | |
| "grad_norm": 2.137456730046166, | |
| "kl": 0.05419921875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0122, | |
| "reward": 0.7231975197792053, | |
| "reward_std": 0.2408137172460556, | |
| "rewards/format_reward": 0.15274719893932343, | |
| "rewards/judgement_reward": 0.5451754331588745, | |
| "rewards/token_reward": 0.025274725630879402, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.81318664550781, | |
| "epoch": 0.6048780487804878, | |
| "grad_norm": 1.8602453492738675, | |
| "kl": 0.03662109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0347, | |
| "reward": 0.9557329416275024, | |
| "reward_std": 0.21061010658740997, | |
| "rewards/format_reward": 0.17472520470619202, | |
| "rewards/judgement_reward": 0.7449086904525757, | |
| "rewards/token_reward": 0.0360989011824131, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.18681335449219, | |
| "epoch": 0.6097560975609756, | |
| "grad_norm": 1.8947771050267905, | |
| "kl": 0.04736328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0427, | |
| "reward": 0.8908948302268982, | |
| "reward_std": 0.21556268632411957, | |
| "rewards/format_reward": 0.16703291237354279, | |
| "rewards/judgement_reward": 0.688202440738678, | |
| "rewards/token_reward": 0.035659339278936386, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.83516693115234, | |
| "epoch": 0.6146341463414634, | |
| "grad_norm": 1.99350313429728, | |
| "kl": 0.0693359375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0231, | |
| "reward": 0.8506399393081665, | |
| "reward_std": 0.24161569774150848, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.6592661738395691, | |
| "rewards/token_reward": 0.026538461446762085, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.53845977783203, | |
| "epoch": 0.6195121951219512, | |
| "grad_norm": 1.7998816564546938, | |
| "kl": 0.0361328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1204, | |
| "reward": 1.039487600326538, | |
| "reward_std": 0.2013106495141983, | |
| "rewards/format_reward": 0.17912080883979797, | |
| "rewards/judgement_reward": 0.8210808038711548, | |
| "rewards/token_reward": 0.03928571194410324, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 81.9120864868164, | |
| "epoch": 0.624390243902439, | |
| "grad_norm": 1.8452835234198748, | |
| "kl": 0.0390625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.12, | |
| "reward": 0.9250213503837585, | |
| "reward_std": 0.2341793179512024, | |
| "rewards/format_reward": 0.17142850160598755, | |
| "rewards/judgement_reward": 0.7072190642356873, | |
| "rewards/token_reward": 0.046373624354600906, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.82967376708984, | |
| "epoch": 0.6292682926829268, | |
| "grad_norm": 1.9489042143231339, | |
| "kl": 0.0311279296875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0785, | |
| "reward": 0.994548499584198, | |
| "reward_std": 0.21068567037582397, | |
| "rewards/format_reward": 0.18021972477436066, | |
| "rewards/judgement_reward": 0.7781199812889099, | |
| "rewards/token_reward": 0.03620879352092743, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.79670715332031, | |
| "epoch": 0.6341463414634146, | |
| "grad_norm": 1.9478743633543802, | |
| "kl": 0.041259765625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0111, | |
| "reward": 0.8490137457847595, | |
| "reward_std": 0.2522919774055481, | |
| "rewards/format_reward": 0.16373620927333832, | |
| "rewards/judgement_reward": 0.6532995104789734, | |
| "rewards/token_reward": 0.031978022307157516, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 81.68681335449219, | |
| "epoch": 0.6390243902439025, | |
| "grad_norm": 2.085728564809279, | |
| "kl": 0.04150390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.074, | |
| "reward": 0.5961366295814514, | |
| "reward_std": 0.2355279177427292, | |
| "rewards/format_reward": 0.1439560055732727, | |
| "rewards/judgement_reward": 0.4331147074699402, | |
| "rewards/token_reward": 0.01906593330204487, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.93406677246094, | |
| "epoch": 0.6439024390243903, | |
| "grad_norm": 1.9504606440875714, | |
| "kl": 0.03857421875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0681, | |
| "reward": 0.9742907881736755, | |
| "reward_std": 0.21939148008823395, | |
| "rewards/format_reward": 0.17802190780639648, | |
| "rewards/judgement_reward": 0.7539612650871277, | |
| "rewards/token_reward": 0.042307693511247635, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.23077392578125, | |
| "epoch": 0.6487804878048781, | |
| "grad_norm": 2.1860995545870168, | |
| "kl": 0.10595703125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0478, | |
| "reward": 0.8597061634063721, | |
| "reward_std": 0.1860429346561432, | |
| "rewards/format_reward": 0.16043950617313385, | |
| "rewards/judgement_reward": 0.6846511363983154, | |
| "rewards/token_reward": 0.014615383930504322, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.5824203491211, | |
| "epoch": 0.6536585365853659, | |
| "grad_norm": 2.056976443684514, | |
| "kl": 0.04736328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0996, | |
| "reward": 1.0542576313018799, | |
| "reward_std": 0.1535356342792511, | |
| "rewards/format_reward": 0.18241751194000244, | |
| "rewards/judgement_reward": 0.8472796678543091, | |
| "rewards/token_reward": 0.024560438469052315, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.73077392578125, | |
| "epoch": 0.6585365853658537, | |
| "grad_norm": 2.354640541368104, | |
| "kl": 0.034912109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0639, | |
| "reward": 1.033756971359253, | |
| "reward_std": 0.12462829798460007, | |
| "rewards/format_reward": 0.18241751194000244, | |
| "rewards/judgement_reward": 0.8278777003288269, | |
| "rewards/token_reward": 0.023461539298295975, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.63736724853516, | |
| "epoch": 0.6634146341463415, | |
| "grad_norm": 1.8711434115369097, | |
| "kl": 0.2275390625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0747, | |
| "reward": 0.8162251710891724, | |
| "reward_std": 0.23913460969924927, | |
| "rewards/format_reward": 0.16703291237354279, | |
| "rewards/judgement_reward": 0.6229832768440247, | |
| "rewards/token_reward": 0.02620879001915455, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.0054931640625, | |
| "epoch": 0.6682926829268293, | |
| "grad_norm": 2.170766069792843, | |
| "kl": 0.158203125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0001, | |
| "reward": 0.5743477940559387, | |
| "reward_std": 0.2055026888847351, | |
| "rewards/format_reward": 0.13736261427402496, | |
| "rewards/judgement_reward": 0.4148424565792084, | |
| "rewards/token_reward": 0.02214285545051098, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.80220031738281, | |
| "epoch": 0.6731707317073171, | |
| "grad_norm": 2.249511172978621, | |
| "kl": 0.040771484375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0299, | |
| "reward": 0.9537181258201599, | |
| "reward_std": 0.2547648847103119, | |
| "rewards/format_reward": 0.17802190780639648, | |
| "rewards/judgement_reward": 0.7325092554092407, | |
| "rewards/token_reward": 0.04318681359291077, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 74.71428680419922, | |
| "epoch": 0.6780487804878049, | |
| "grad_norm": 2.4468786224687973, | |
| "kl": 0.39453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0937, | |
| "reward": 0.8086802959442139, | |
| "reward_std": 0.2142382711172104, | |
| "rewards/format_reward": 0.15824170410633087, | |
| "rewards/judgement_reward": 0.6207680702209473, | |
| "rewards/token_reward": 0.02967032790184021, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 71.93955993652344, | |
| "epoch": 0.6829268292682927, | |
| "grad_norm": 2.5243974633393496, | |
| "kl": 0.046630859375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1273, | |
| "reward": 0.9722238183021545, | |
| "reward_std": 0.18802900612354279, | |
| "rewards/format_reward": 0.18131862580776215, | |
| "rewards/judgement_reward": 0.768817126750946, | |
| "rewards/token_reward": 0.022087909281253815, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.4835205078125, | |
| "epoch": 0.6878048780487804, | |
| "grad_norm": 2.7374184261837455, | |
| "kl": 0.05029296875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0257, | |
| "reward": 0.9533888101577759, | |
| "reward_std": 0.19217993319034576, | |
| "rewards/format_reward": 0.17362630367279053, | |
| "rewards/judgement_reward": 0.7552569508552551, | |
| "rewards/token_reward": 0.02450549229979515, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.15384674072266, | |
| "epoch": 0.6926829268292682, | |
| "grad_norm": 1.882857912628275, | |
| "kl": 0.048583984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1109, | |
| "reward": 0.9248565435409546, | |
| "reward_std": 0.22721460461616516, | |
| "rewards/format_reward": 0.17032961547374725, | |
| "rewards/judgement_reward": 0.7072190642356873, | |
| "rewards/token_reward": 0.04730769246816635, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 81.5879135131836, | |
| "epoch": 0.697560975609756, | |
| "grad_norm": 1.9696456406180098, | |
| "kl": 0.0498046875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0738, | |
| "reward": 1.0071877241134644, | |
| "reward_std": 0.24609951674938202, | |
| "rewards/format_reward": 0.18021969497203827, | |
| "rewards/judgement_reward": 0.775978684425354, | |
| "rewards/token_reward": 0.05098900571465492, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.44506072998047, | |
| "epoch": 0.7024390243902439, | |
| "grad_norm": 1.9874995933520714, | |
| "kl": 0.046142578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0414, | |
| "reward": 0.9923238754272461, | |
| "reward_std": 0.2381928414106369, | |
| "rewards/format_reward": 0.17252740263938904, | |
| "rewards/judgement_reward": 0.7843018770217896, | |
| "rewards/token_reward": 0.03549450263381004, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.11538696289062, | |
| "epoch": 0.7073170731707317, | |
| "grad_norm": 2.170568236496205, | |
| "kl": 0.044921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0286, | |
| "reward": 1.019083857536316, | |
| "reward_std": 0.2215217649936676, | |
| "rewards/format_reward": 0.18131862580776215, | |
| "rewards/judgement_reward": 0.8043037056922913, | |
| "rewards/token_reward": 0.03346153721213341, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 81.14286041259766, | |
| "epoch": 0.7121951219512195, | |
| "grad_norm": 1.9765889361911162, | |
| "kl": 0.056640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0658, | |
| "reward": 0.6368016004562378, | |
| "reward_std": 0.19664892554283142, | |
| "rewards/format_reward": 0.1450549066066742, | |
| "rewards/judgement_reward": 0.4633399248123169, | |
| "rewards/token_reward": 0.028406593948602676, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.33516693115234, | |
| "epoch": 0.7170731707317073, | |
| "grad_norm": 1.749209988774624, | |
| "kl": 0.0537109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0663, | |
| "reward": 0.962835967540741, | |
| "reward_std": 0.2308100312948227, | |
| "rewards/format_reward": 0.17252741754055023, | |
| "rewards/judgement_reward": 0.7419567704200745, | |
| "rewards/token_reward": 0.04835164546966553, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.8901138305664, | |
| "epoch": 0.7219512195121951, | |
| "grad_norm": 2.0335137757386077, | |
| "kl": 0.091796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0357, | |
| "reward": 0.9629313945770264, | |
| "reward_std": 0.20034556090831757, | |
| "rewards/format_reward": 0.18131862580776215, | |
| "rewards/judgement_reward": 0.7572720646858215, | |
| "rewards/token_reward": 0.024340655654668808, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.15933990478516, | |
| "epoch": 0.7268292682926829, | |
| "grad_norm": 1.7647654238864119, | |
| "kl": 0.051025390625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0586, | |
| "reward": 0.9167026877403259, | |
| "reward_std": 0.1917964667081833, | |
| "rewards/format_reward": 0.16373620927333832, | |
| "rewards/judgement_reward": 0.7133510112762451, | |
| "rewards/token_reward": 0.03961538150906563, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.3901138305664, | |
| "epoch": 0.7317073170731707, | |
| "grad_norm": 1.9248149745795777, | |
| "kl": 0.047119140625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1216, | |
| "reward": 0.8943402767181396, | |
| "reward_std": 0.21739540994167328, | |
| "rewards/format_reward": 0.16813181340694427, | |
| "rewards/judgement_reward": 0.6763730645179749, | |
| "rewards/token_reward": 0.04983516409993172, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.5714340209961, | |
| "epoch": 0.7365853658536585, | |
| "grad_norm": 2.0080133802993836, | |
| "kl": 0.056884765625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0591, | |
| "reward": 0.967893660068512, | |
| "reward_std": 0.24371013045310974, | |
| "rewards/format_reward": 0.17252740263938904, | |
| "rewards/judgement_reward": 0.7508604526519775, | |
| "rewards/token_reward": 0.04450549930334091, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.76374053955078, | |
| "epoch": 0.7414634146341463, | |
| "grad_norm": 1.923607313408308, | |
| "kl": 0.044189453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0686, | |
| "reward": 0.9229432940483093, | |
| "reward_std": 0.23856668174266815, | |
| "rewards/format_reward": 0.16923069953918457, | |
| "rewards/judgement_reward": 0.7147566676139832, | |
| "rewards/token_reward": 0.03895604610443115, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.60989379882812, | |
| "epoch": 0.7463414634146341, | |
| "grad_norm": 1.9096459984701761, | |
| "kl": 0.043212890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0566, | |
| "reward": 1.0493299961090088, | |
| "reward_std": 0.22466498613357544, | |
| "rewards/format_reward": 0.18461531400680542, | |
| "rewards/judgement_reward": 0.8225165605545044, | |
| "rewards/token_reward": 0.04219780117273331, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.5988998413086, | |
| "epoch": 0.751219512195122, | |
| "grad_norm": 1.9835076997357493, | |
| "kl": 0.04150390625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0931, | |
| "reward": 0.8906086683273315, | |
| "reward_std": 0.14505840837955475, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.7020372152328491, | |
| "rewards/token_reward": 0.023736264556646347, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.36264038085938, | |
| "epoch": 0.7560975609756098, | |
| "grad_norm": 1.840974141351389, | |
| "kl": 0.044921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.091, | |
| "reward": 1.1236339807510376, | |
| "reward_std": 0.17279954254627228, | |
| "rewards/format_reward": 0.1868131309747696, | |
| "rewards/judgement_reward": 0.8986337780952454, | |
| "rewards/token_reward": 0.03818681463599205, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.02747344970703, | |
| "epoch": 0.7609756097560976, | |
| "grad_norm": 1.963792827769517, | |
| "kl": 0.049560546875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0361, | |
| "reward": 0.855387270450592, | |
| "reward_std": 0.22867831587791443, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.6642332077026367, | |
| "rewards/token_reward": 0.029615381732583046, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.95055389404297, | |
| "epoch": 0.7658536585365854, | |
| "grad_norm": 1.9306677990628487, | |
| "kl": 0.04736328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0329, | |
| "reward": 0.9254786372184753, | |
| "reward_std": 0.23398159444332123, | |
| "rewards/format_reward": 0.17032961547374725, | |
| "rewards/judgement_reward": 0.7126215100288391, | |
| "rewards/token_reward": 0.042527470737695694, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.71428680419922, | |
| "epoch": 0.7707317073170732, | |
| "grad_norm": 1.9488316654243145, | |
| "kl": 0.04541015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0659, | |
| "reward": 1.0084419250488281, | |
| "reward_std": 0.18995091319084167, | |
| "rewards/format_reward": 0.18351641297340393, | |
| "rewards/judgement_reward": 0.7943759560585022, | |
| "rewards/token_reward": 0.03054944798350334, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.46154022216797, | |
| "epoch": 0.775609756097561, | |
| "grad_norm": 2.4866962371762926, | |
| "kl": 0.107421875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0118, | |
| "reward": 1.0291651487350464, | |
| "reward_std": 0.22909829020500183, | |
| "rewards/format_reward": 0.17032960057258606, | |
| "rewards/judgement_reward": 0.8225165605545044, | |
| "rewards/token_reward": 0.03631868213415146, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.51099395751953, | |
| "epoch": 0.7804878048780488, | |
| "grad_norm": 2.1399222908924966, | |
| "kl": 0.035888671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1147, | |
| "reward": 1.0689184665679932, | |
| "reward_std": 0.2075626105070114, | |
| "rewards/format_reward": 0.17912080883979797, | |
| "rewards/judgement_reward": 0.8432589173316956, | |
| "rewards/token_reward": 0.04653845727443695, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.28571319580078, | |
| "epoch": 0.7853658536585366, | |
| "grad_norm": 2.0654519129717954, | |
| "kl": 0.038818359375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0747, | |
| "reward": 1.0535253286361694, | |
| "reward_std": 0.1914973109960556, | |
| "rewards/format_reward": 0.18021972477436066, | |
| "rewards/judgement_reward": 0.8373165726661682, | |
| "rewards/token_reward": 0.03598900884389877, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 75.79121398925781, | |
| "epoch": 0.7902439024390244, | |
| "grad_norm": 1.8597487323795248, | |
| "kl": 0.037353515625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0976, | |
| "reward": 0.981767475605011, | |
| "reward_std": 0.15387368202209473, | |
| "rewards/format_reward": 0.17142850160598755, | |
| "rewards/judgement_reward": 0.7899540662765503, | |
| "rewards/token_reward": 0.020384615287184715, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.78571319580078, | |
| "epoch": 0.7951219512195122, | |
| "grad_norm": 1.8877094518624145, | |
| "kl": 0.0517578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0441, | |
| "reward": 0.611198365688324, | |
| "reward_std": 0.23072822391986847, | |
| "rewards/format_reward": 0.14065930247306824, | |
| "rewards/judgement_reward": 0.431198388338089, | |
| "rewards/token_reward": 0.03934066370129585, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.0054931640625, | |
| "epoch": 0.8, | |
| "grad_norm": 1.7914504262564466, | |
| "kl": 0.049072265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0773, | |
| "reward": 1.0531319379806519, | |
| "reward_std": 0.203482985496521, | |
| "rewards/format_reward": 0.1758241206407547, | |
| "rewards/judgement_reward": 0.8310439586639404, | |
| "rewards/token_reward": 0.04626372829079628, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.20330047607422, | |
| "epoch": 0.8048780487804879, | |
| "grad_norm": 1.8591700696334572, | |
| "kl": 0.03662109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0489, | |
| "reward": 1.1166560649871826, | |
| "reward_std": 0.1697649508714676, | |
| "rewards/format_reward": 0.1857142299413681, | |
| "rewards/judgement_reward": 0.8986338973045349, | |
| "rewards/token_reward": 0.032307688146829605, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.22527313232422, | |
| "epoch": 0.8097560975609757, | |
| "grad_norm": 2.214129107678454, | |
| "kl": 0.052978515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0263, | |
| "reward": 0.6829060912132263, | |
| "reward_std": 0.25708603858947754, | |
| "rewards/format_reward": 0.14285710453987122, | |
| "rewards/judgement_reward": 0.5113676190376282, | |
| "rewards/token_reward": 0.0286813173443079, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.74725341796875, | |
| "epoch": 0.8146341463414634, | |
| "grad_norm": 1.8131229003189935, | |
| "kl": 0.048095703125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0818, | |
| "reward": 0.8743994832038879, | |
| "reward_std": 0.16046012938022614, | |
| "rewards/format_reward": 0.1648351103067398, | |
| "rewards/judgement_reward": 0.672916054725647, | |
| "rewards/token_reward": 0.03664834797382355, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.55494689941406, | |
| "epoch": 0.8195121951219512, | |
| "grad_norm": 1.8955641226851798, | |
| "kl": 0.0556640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0973, | |
| "reward": 1.104522705078125, | |
| "reward_std": 0.19338937103748322, | |
| "rewards/format_reward": 0.18241751194000244, | |
| "rewards/judgement_reward": 0.8802368640899658, | |
| "rewards/token_reward": 0.04186813160777092, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 84.85165405273438, | |
| "epoch": 0.824390243902439, | |
| "grad_norm": 1.7813215626916248, | |
| "kl": 0.05029296875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0779, | |
| "reward": 0.8853285312652588, | |
| "reward_std": 0.22625528275966644, | |
| "rewards/format_reward": 0.1659340113401413, | |
| "rewards/judgement_reward": 0.6722515225410461, | |
| "rewards/token_reward": 0.04714285209774971, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.70879364013672, | |
| "epoch": 0.8292682926829268, | |
| "grad_norm": 1.8224025120445444, | |
| "kl": 0.05224609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1272, | |
| "reward": 1.0830038785934448, | |
| "reward_std": 0.1721232831478119, | |
| "rewards/format_reward": 0.18131862580776215, | |
| "rewards/judgement_reward": 0.858058750629425, | |
| "rewards/token_reward": 0.043626368045806885, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.5934066772461, | |
| "epoch": 0.8341463414634146, | |
| "grad_norm": 1.8203230311858847, | |
| "kl": 0.050048828125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1045, | |
| "reward": 1.1872518062591553, | |
| "reward_std": 0.12512442469596863, | |
| "rewards/format_reward": 0.1879120171070099, | |
| "rewards/judgement_reward": 0.9585154056549072, | |
| "rewards/token_reward": 0.040824174880981445, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.5988998413086, | |
| "epoch": 0.8390243902439024, | |
| "grad_norm": 1.8580888818259809, | |
| "kl": 0.06591796875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.012, | |
| "reward": 0.7364369034767151, | |
| "reward_std": 0.24575328826904297, | |
| "rewards/format_reward": 0.14945051074028015, | |
| "rewards/judgement_reward": 0.5472610592842102, | |
| "rewards/token_reward": 0.039725273847579956, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.8901138305664, | |
| "epoch": 0.8439024390243902, | |
| "grad_norm": 1.9076522374281615, | |
| "kl": 0.0673828125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0474, | |
| "reward": 0.700067400932312, | |
| "reward_std": 0.17015255987644196, | |
| "rewards/format_reward": 0.15714280307292938, | |
| "rewards/judgement_reward": 0.5104518532752991, | |
| "rewards/token_reward": 0.03247252479195595, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 81.93406677246094, | |
| "epoch": 0.848780487804878, | |
| "grad_norm": 1.9086822135882167, | |
| "kl": 0.06396484375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.099, | |
| "reward": 0.841740071773529, | |
| "reward_std": 0.18049749732017517, | |
| "rewards/format_reward": 0.16263730823993683, | |
| "rewards/judgement_reward": 0.6345422863960266, | |
| "rewards/token_reward": 0.04456043988466263, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.34066009521484, | |
| "epoch": 0.8536585365853658, | |
| "grad_norm": 1.8635805588199053, | |
| "kl": 0.055419921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0344, | |
| "reward": 0.9539201855659485, | |
| "reward_std": 0.20264722406864166, | |
| "rewards/format_reward": 0.16813179850578308, | |
| "rewards/judgement_reward": 0.7532058954238892, | |
| "rewards/token_reward": 0.03258241340517998, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.26374053955078, | |
| "epoch": 0.8585365853658536, | |
| "grad_norm": 2.0330839117251336, | |
| "kl": 0.361328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0225, | |
| "reward": 0.918322741985321, | |
| "reward_std": 0.2709953188896179, | |
| "rewards/format_reward": 0.17802190780639648, | |
| "rewards/judgement_reward": 0.706509530544281, | |
| "rewards/token_reward": 0.033791206777095795, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.0824203491211, | |
| "epoch": 0.8634146341463415, | |
| "grad_norm": 1.7553145835440749, | |
| "kl": 0.04833984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0939, | |
| "reward": 1.1063908338546753, | |
| "reward_std": 0.17723402380943298, | |
| "rewards/format_reward": 0.18241752684116364, | |
| "rewards/judgement_reward": 0.8802370429039001, | |
| "rewards/token_reward": 0.04373626038432121, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.05494689941406, | |
| "epoch": 0.8682926829268293, | |
| "grad_norm": 1.7701802023624165, | |
| "kl": 0.0478515625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0524, | |
| "reward": 1.1443761587142944, | |
| "reward_std": 0.151431143283844, | |
| "rewards/format_reward": 0.18791203200817108, | |
| "rewards/judgement_reward": 0.9193762540817261, | |
| "rewards/token_reward": 0.03708790987730026, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 76.87911987304688, | |
| "epoch": 0.8731707317073171, | |
| "grad_norm": 1.948906301032531, | |
| "kl": 0.04833984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0406, | |
| "reward": 1.0914630889892578, | |
| "reward_std": 0.17707206308841705, | |
| "rewards/format_reward": 0.1857142299413681, | |
| "rewards/judgement_reward": 0.8778916597366333, | |
| "rewards/token_reward": 0.027857141569256783, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.11538696289062, | |
| "epoch": 0.8780487804878049, | |
| "grad_norm": 1.9707362441929812, | |
| "kl": 0.041748046875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0719, | |
| "reward": 0.8997630476951599, | |
| "reward_std": 0.19621287286281586, | |
| "rewards/format_reward": 0.17252741754055023, | |
| "rewards/judgement_reward": 0.6895432472229004, | |
| "rewards/token_reward": 0.03769230842590332, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.06593322753906, | |
| "epoch": 0.8829268292682927, | |
| "grad_norm": 2.1153280959093315, | |
| "kl": 0.04931640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1035, | |
| "reward": 0.8650724291801453, | |
| "reward_std": 0.18447832763195038, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.6753472089767456, | |
| "rewards/token_reward": 0.02818680927157402, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.80220031738281, | |
| "epoch": 0.8878048780487805, | |
| "grad_norm": 1.8505179859150593, | |
| "kl": 0.10546875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1083, | |
| "reward": 1.0732321739196777, | |
| "reward_std": 0.15069392323493958, | |
| "rewards/format_reward": 0.18131862580776215, | |
| "rewards/judgement_reward": 0.8640012145042419, | |
| "rewards/token_reward": 0.027912087738513947, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.72527313232422, | |
| "epoch": 0.8926829268292683, | |
| "grad_norm": 1.7933498219789568, | |
| "kl": 0.1640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0539, | |
| "reward": 1.0424714088439941, | |
| "reward_std": 0.25467538833618164, | |
| "rewards/format_reward": 0.18351641297340393, | |
| "rewards/judgement_reward": 0.8041195869445801, | |
| "rewards/token_reward": 0.05483516305685043, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.67582702636719, | |
| "epoch": 0.8975609756097561, | |
| "grad_norm": 2.600363026488421, | |
| "kl": 0.04931640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0476, | |
| "reward": 0.755411684513092, | |
| "reward_std": 0.23074375092983246, | |
| "rewards/format_reward": 0.15714281797409058, | |
| "rewards/judgement_reward": 0.5580490231513977, | |
| "rewards/token_reward": 0.040219780057668686, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 84.43955993652344, | |
| "epoch": 0.9024390243902439, | |
| "grad_norm": 1.7627204492703208, | |
| "kl": 0.0537109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0844, | |
| "reward": 1.1599979400634766, | |
| "reward_std": 0.17514334619045258, | |
| "rewards/format_reward": 0.18901091814041138, | |
| "rewards/judgement_reward": 0.9170307517051697, | |
| "rewards/token_reward": 0.0539560429751873, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 78.25824737548828, | |
| "epoch": 0.9073170731707317, | |
| "grad_norm": 2.0241371886975035, | |
| "kl": 0.05517578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1234, | |
| "reward": 1.0605839490890503, | |
| "reward_std": 0.17436476051807404, | |
| "rewards/format_reward": 0.18131862580776215, | |
| "rewards/judgement_reward": 0.8380013704299927, | |
| "rewards/token_reward": 0.04126373305916786, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 83.9011001586914, | |
| "epoch": 0.9121951219512195, | |
| "grad_norm": 1.7110207760725646, | |
| "kl": 0.055419921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0842, | |
| "reward": 0.7558942437171936, | |
| "reward_std": 0.18241019546985626, | |
| "rewards/format_reward": 0.1549450010061264, | |
| "rewards/judgement_reward": 0.5489163398742676, | |
| "rewards/token_reward": 0.052032966166734695, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.96154022216797, | |
| "epoch": 0.9170731707317074, | |
| "grad_norm": 1.7123226348250025, | |
| "kl": 0.26171875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1222, | |
| "reward": 0.8997620940208435, | |
| "reward_std": 0.2118684947490692, | |
| "rewards/format_reward": 0.17142850160598755, | |
| "rewards/judgement_reward": 0.6890478134155273, | |
| "rewards/token_reward": 0.03928571194410324, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.17033386230469, | |
| "epoch": 0.9219512195121952, | |
| "grad_norm": 1.830366489112101, | |
| "kl": 0.060302734375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0873, | |
| "reward": 1.1136815547943115, | |
| "reward_std": 0.16834889352321625, | |
| "rewards/format_reward": 0.18131859600543976, | |
| "rewards/judgement_reward": 0.897197961807251, | |
| "rewards/token_reward": 0.035164833068847656, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 85.31318664550781, | |
| "epoch": 0.926829268292683, | |
| "grad_norm": 1.6347844224064585, | |
| "kl": 0.059814453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0761, | |
| "reward": 1.1308867931365967, | |
| "reward_std": 0.17480330169200897, | |
| "rewards/format_reward": 0.18241752684116364, | |
| "rewards/judgement_reward": 0.8986338973045349, | |
| "rewards/token_reward": 0.04983516409993172, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 84.97802734375, | |
| "epoch": 0.9317073170731708, | |
| "grad_norm": 1.6920004921526572, | |
| "kl": 0.05615234375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0895, | |
| "reward": 1.003623604774475, | |
| "reward_std": 0.17075157165527344, | |
| "rewards/format_reward": 0.17472520470619202, | |
| "rewards/judgement_reward": 0.7810961008071899, | |
| "rewards/token_reward": 0.04780219867825508, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 84.5714340209961, | |
| "epoch": 0.9365853658536586, | |
| "grad_norm": 2.022501703444944, | |
| "kl": 0.08837890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.039, | |
| "reward": 0.8640207648277283, | |
| "reward_std": 0.23843564093112946, | |
| "rewards/format_reward": 0.16043950617313385, | |
| "rewards/judgement_reward": 0.6474823355674744, | |
| "rewards/token_reward": 0.056098904460668564, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 84.76923370361328, | |
| "epoch": 0.9414634146341463, | |
| "grad_norm": 1.6540302758951702, | |
| "kl": 0.058349609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0698, | |
| "reward": 1.0908318758010864, | |
| "reward_std": 0.18835541605949402, | |
| "rewards/format_reward": 0.18241751194000244, | |
| "rewards/judgement_reward": 0.8616559505462646, | |
| "rewards/token_reward": 0.046758245676755905, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 86.17033386230469, | |
| "epoch": 0.9463414634146341, | |
| "grad_norm": 2.029218121152698, | |
| "kl": 0.06591796875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0222, | |
| "reward": 0.7869437336921692, | |
| "reward_std": 0.28270089626312256, | |
| "rewards/format_reward": 0.16263730823993683, | |
| "rewards/judgement_reward": 0.5819985866546631, | |
| "rewards/token_reward": 0.042307689785957336, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.43955993652344, | |
| "epoch": 0.9512195121951219, | |
| "grad_norm": 1.9170587116508577, | |
| "kl": 0.061767578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0761, | |
| "reward": 1.1304645538330078, | |
| "reward_std": 0.1845540553331375, | |
| "rewards/format_reward": 0.18791203200817108, | |
| "rewards/judgement_reward": 0.8962885737419128, | |
| "rewards/token_reward": 0.046263739466667175, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 85.04396057128906, | |
| "epoch": 0.9560975609756097, | |
| "grad_norm": 1.7642610809126764, | |
| "kl": 0.053466796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0974, | |
| "reward": 1.0226733684539795, | |
| "reward_std": 0.23680001497268677, | |
| "rewards/format_reward": 0.18241751194000244, | |
| "rewards/judgement_reward": 0.7795965075492859, | |
| "rewards/token_reward": 0.06065933778882027, | |
| "step": 196 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.42857360839844, | |
| "epoch": 0.9609756097560975, | |
| "grad_norm": 1.8138413422032649, | |
| "kl": 0.05712890625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0532, | |
| "reward": 0.8977496027946472, | |
| "reward_std": 0.20727433264255524, | |
| "rewards/format_reward": 0.17032960057258606, | |
| "rewards/judgement_reward": 0.6894528269767761, | |
| "rewards/token_reward": 0.03796703368425369, | |
| "step": 197 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 86.25274658203125, | |
| "epoch": 0.9658536585365853, | |
| "grad_norm": 1.896125990017999, | |
| "kl": 0.053955078125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0677, | |
| "reward": 0.8915479183197021, | |
| "reward_std": 0.2226872593164444, | |
| "rewards/format_reward": 0.16153840720653534, | |
| "rewards/judgement_reward": 0.6870973110198975, | |
| "rewards/token_reward": 0.042912084609270096, | |
| "step": 198 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.41758728027344, | |
| "epoch": 0.9707317073170731, | |
| "grad_norm": 1.854570297914855, | |
| "kl": 0.048828125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0345, | |
| "reward": 0.7600922584533691, | |
| "reward_std": 0.22392849624156952, | |
| "rewards/format_reward": 0.1560439020395279, | |
| "rewards/judgement_reward": 0.575421929359436, | |
| "rewards/token_reward": 0.028626374900341034, | |
| "step": 199 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 82.27472686767578, | |
| "epoch": 0.975609756097561, | |
| "grad_norm": 1.8461878166271504, | |
| "kl": 0.05322265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0727, | |
| "reward": 0.5569639205932617, | |
| "reward_std": 0.14818796515464783, | |
| "rewards/format_reward": 0.1340659111738205, | |
| "rewards/judgement_reward": 0.3882276713848114, | |
| "rewards/token_reward": 0.034670326858758926, | |
| "step": 200 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 79.75824737548828, | |
| "epoch": 0.9804878048780488, | |
| "grad_norm": 2.318927168904253, | |
| "kl": 0.056396484375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0058, | |
| "reward": 0.5673214197158813, | |
| "reward_std": 0.23380212485790253, | |
| "rewards/format_reward": 0.14285710453987122, | |
| "rewards/judgement_reward": 0.393749862909317, | |
| "rewards/token_reward": 0.030714284628629684, | |
| "step": 201 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.66484069824219, | |
| "epoch": 0.9853658536585366, | |
| "grad_norm": 1.9516131221905255, | |
| "kl": 0.051513671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.1456, | |
| "reward": 1.2027462720870972, | |
| "reward_std": 0.13449910283088684, | |
| "rewards/format_reward": 0.19230760633945465, | |
| "rewards/judgement_reward": 0.9585154056549072, | |
| "rewards/token_reward": 0.051923077553510666, | |
| "step": 202 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 77.9065933227539, | |
| "epoch": 0.9902439024390244, | |
| "grad_norm": 1.9128695892010321, | |
| "kl": 0.052734375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0288, | |
| "reward": 0.8996109366416931, | |
| "reward_std": 0.25487908720970154, | |
| "rewards/format_reward": 0.17362631857395172, | |
| "rewards/judgement_reward": 0.7061491012573242, | |
| "rewards/token_reward": 0.01983516290783882, | |
| "step": 203 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.14286041259766, | |
| "epoch": 0.9951219512195122, | |
| "grad_norm": 1.7460376924567018, | |
| "kl": 0.058349609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0686, | |
| "reward": 0.9356229305267334, | |
| "reward_std": 0.17833828926086426, | |
| "rewards/format_reward": 0.17032961547374725, | |
| "rewards/judgement_reward": 0.7273810505867004, | |
| "rewards/token_reward": 0.03791208565235138, | |
| "step": 204 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 80.55555725097656, | |
| "epoch": 1.0, | |
| "grad_norm": 1.8130933404400773, | |
| "kl": 0.051513671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0604, | |
| "reward": 0.9366883635520935, | |
| "reward_std": 0.1746482402086258, | |
| "rewards/format_reward": 0.1560439020395279, | |
| "rewards/judgement_reward": 0.7467983365058899, | |
| "rewards/token_reward": 0.03384615480899811, | |
| "step": 205 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 205, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 26, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |