{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 927,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 0.333984375,
      "learning_rate": 5.3763440860215056e-09,
      "logits/chosen": -1.7726776599884033,
      "logits/rejected": -1.019553542137146,
      "logps/chosen": -227.8472900390625,
      "logps/rejected": -244.70220947265625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/margins_max": 0.0,
      "rewards/margins_min": 0.0,
      "rewards/margins_std": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 0.33203125,
      "learning_rate": 5.3763440860215054e-08,
      "logits/chosen": -1.2758857011795044,
      "logits/rejected": -0.7481470108032227,
      "logps/chosen": -294.4023132324219,
      "logps/rejected": -209.6774139404297,
      "loss": 0.6931,
      "rewards/accuracies": 0.4722222089767456,
      "rewards/chosen": 0.00042969180503860116,
      "rewards/margins": 0.00026647368213161826,
      "rewards/margins_max": 0.002202093368396163,
      "rewards/margins_min": -0.0016691461205482483,
      "rewards/margins_std": 0.002737379400059581,
      "rewards/rejected": 0.00016321813745889813,
      "step": 10
    },
    {
      "epoch": 0.02,
      "grad_norm": 0.3515625,
      "learning_rate": 1.0752688172043011e-07,
      "logits/chosen": -1.4951783418655396,
      "logits/rejected": -1.0168498754501343,
      "logps/chosen": -280.9791259765625,
      "logps/rejected": -274.51055908203125,
      "loss": 0.6928,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0005079669645056129,
      "rewards/margins": 0.0006722012185491621,
      "rewards/margins_max": 0.003316085785627365,
      "rewards/margins_min": -0.001971683232113719,
      "rewards/margins_std": 0.003739017527550459,
      "rewards/rejected": -0.00016423416673205793,
      "step": 20
    },
    {
      "epoch": 0.03,
      "grad_norm": 0.2734375,
      "learning_rate": 1.6129032258064515e-07,
      "logits/chosen": -1.1558444499969482,
      "logits/rejected": -0.7869107723236084,
      "logps/chosen": -235.8018035888672,
      "logps/rejected": -239.435791015625,
      "loss": 0.6926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0022618253715336323,
      "rewards/margins": 0.001511804643087089,
      "rewards/margins_max": 0.004373847972601652,
      "rewards/margins_min": -0.001350238686427474,
      "rewards/margins_std": 0.004047540482133627,
      "rewards/rejected": 0.000750020903069526,
      "step": 30
    },
    {
      "epoch": 0.04,
      "grad_norm": 0.2421875,
      "learning_rate": 2.1505376344086022e-07,
      "logits/chosen": -1.4029566049575806,
      "logits/rejected": -0.8758159875869751,
      "logps/chosen": -256.91668701171875,
      "logps/rejected": -245.5706024169922,
      "loss": 0.6927,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0011851616436615586,
      "rewards/margins": 0.0004023304209113121,
      "rewards/margins_max": 0.002305095549672842,
      "rewards/margins_min": -0.0015004349406808615,
      "rewards/margins_std": 0.0026909164153039455,
      "rewards/rejected": 0.0007828312227502465,
      "step": 40
    },
    {
      "epoch": 0.05,
      "grad_norm": 0.25390625,
      "learning_rate": 2.6881720430107523e-07,
      "logits/chosen": -1.5180784463882446,
      "logits/rejected": -1.0010168552398682,
      "logps/chosen": -202.52719116210938,
      "logps/rejected": -197.68511962890625,
      "loss": 0.6926,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0013484725495800376,
      "rewards/margins": 0.0014005316188558936,
      "rewards/margins_max": 0.0035043410025537014,
      "rewards/margins_min": -0.0007032775320112705,
      "rewards/margins_std": 0.002975235693156719,
      "rewards/rejected": -5.2059163863305e-05,
      "step": 50
    },
    {
      "epoch": 0.06,
      "grad_norm": 0.287109375,
      "learning_rate": 3.225806451612903e-07,
      "logits/chosen": -1.5808765888214111,
      "logits/rejected": -0.893613338470459,
      "logps/chosen": -291.6551208496094,
      "logps/rejected": -255.85360717773438,
      "loss": 0.6922,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0011142367729917169,
      "rewards/margins": 0.0015284843975678086,
      "rewards/margins_max": 0.0042257350869476795,
      "rewards/margins_min": -0.0011687660589814186,
      "rewards/margins_std": 0.003814487950876355,
      "rewards/rejected": -0.0004142475372646004,
      "step": 60
    },
    {
      "epoch": 0.08,
      "grad_norm": 0.296875,
      "learning_rate": 3.7634408602150537e-07,
      "logits/chosen": -1.6458534002304077,
      "logits/rejected": -0.9159662127494812,
      "logps/chosen": -353.4042053222656,
      "logps/rejected": -295.0526428222656,
      "loss": 0.6916,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003586581675335765,
      "rewards/margins": 0.0029636994004249573,
      "rewards/margins_max": 0.0056008645333349705,
      "rewards/margins_min": 0.00032653429661877453,
      "rewards/margins_std": 0.0037295143119990826,
      "rewards/rejected": 0.0006228827987797558,
      "step": 70
    },
    {
      "epoch": 0.09,
      "grad_norm": 0.283203125,
      "learning_rate": 4.3010752688172043e-07,
      "logits/chosen": -1.3944181203842163,
      "logits/rejected": -0.9768520593643188,
      "logps/chosen": -247.8656463623047,
      "logps/rejected": -222.736083984375,
      "loss": 0.691,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004138199612498283,
      "rewards/margins": 0.0038313076365739107,
      "rewards/margins_max": 0.007481383625417948,
      "rewards/margins_min": 0.00018123061454389244,
      "rewards/margins_std": 0.005161988083273172,
      "rewards/rejected": 0.00030689238337799907,
      "step": 80
    },
    {
      "epoch": 0.1,
      "grad_norm": 0.263671875,
      "learning_rate": 4.838709677419355e-07,
      "logits/chosen": -1.4816954135894775,
      "logits/rejected": -1.0011855363845825,
      "logps/chosen": -307.66729736328125,
      "logps/rejected": -218.35110473632812,
      "loss": 0.6901,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.006525175180286169,
      "rewards/margins": 0.006856041494756937,
      "rewards/margins_max": 0.012032730504870415,
      "rewards/margins_min": 0.0016793517861515284,
      "rewards/margins_std": 0.007320943288505077,
      "rewards/rejected": -0.0003308658779133111,
      "step": 90
    },
    {
      "epoch": 0.11,
      "grad_norm": 0.275390625,
      "learning_rate": 4.999130942376231e-07,
      "logits/chosen": -1.4475181102752686,
      "logits/rejected": -0.8290830850601196,
      "logps/chosen": -247.40322875976562,
      "logps/rejected": -220.42355346679688,
      "loss": 0.6897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.007414130959659815,
      "rewards/margins": 0.0059156701900064945,
      "rewards/margins_max": 0.00915153045207262,
      "rewards/margins_min": 0.0026798094622790813,
      "rewards/margins_std": 0.004576197825372219,
      "rewards/rejected": 0.0014984606532379985,
      "step": 100
    },
    {
      "epoch": 0.12,
      "grad_norm": 0.328125,
      "learning_rate": 4.994875788073206e-07,
      "logits/chosen": -1.3422911167144775,
      "logits/rejected": -0.9277170300483704,
      "logps/chosen": -265.04791259765625,
      "logps/rejected": -291.83612060546875,
      "loss": 0.6882,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.009832861833274364,
      "rewards/margins": 0.010079814121127129,
      "rewards/margins_max": 0.014872267842292786,
      "rewards/margins_min": 0.005287360865622759,
      "rewards/margins_std": 0.006777553353458643,
      "rewards/rejected": -0.00024695199681445956,
      "step": 110
    },
    {
      "epoch": 0.13,
      "grad_norm": 0.3515625,
      "learning_rate": 4.987080943856886e-07,
      "logits/chosen": -1.4355990886688232,
      "logits/rejected": -0.9039069414138794,
      "logps/chosen": -241.0117950439453,
      "logps/rejected": -261.8793640136719,
      "loss": 0.6878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.010962730273604393,
      "rewards/margins": 0.011187642812728882,
      "rewards/margins_max": 0.01832672953605652,
      "rewards/margins_min": 0.0040485551580786705,
      "rewards/margins_std": 0.010096193291246891,
      "rewards/rejected": -0.0002249126264359802,
      "step": 120
    },
    {
      "epoch": 0.14,
      "grad_norm": 0.318359375,
      "learning_rate": 4.975757468927726e-07,
      "logits/chosen": -1.5994830131530762,
      "logits/rejected": -0.8840494155883789,
      "logps/chosen": -262.6464538574219,
      "logps/rejected": -225.1462860107422,
      "loss": 0.6878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.01504091639071703,
      "rewards/margins": 0.014881642535328865,
      "rewards/margins_max": 0.021485231816768646,
      "rewards/margins_min": 0.008278051391243935,
      "rewards/margins_std": 0.00933888740837574,
      "rewards/rejected": 0.00015927411732263863,
      "step": 130
    },
    {
      "epoch": 0.15,
      "grad_norm": 0.28125,
      "learning_rate": 4.960921428851066e-07,
      "logits/chosen": -1.3022377490997314,
      "logits/rejected": -0.9370013475418091,
      "logps/chosen": -238.2805938720703,
      "logps/rejected": -267.19854736328125,
      "loss": 0.6864,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.015973640605807304,
      "rewards/margins": 0.014827728271484375,
      "rewards/margins_max": 0.02404005080461502,
      "rewards/margins_min": 0.005615406669676304,
      "rewards/margins_std": 0.013028192333877087,
      "rewards/rejected": 0.001145911985076964,
      "step": 140
    },
    {
      "epoch": 0.16,
      "grad_norm": 0.37890625,
      "learning_rate": 4.942593872763566e-07,
      "logits/chosen": -1.4023020267486572,
      "logits/rejected": -0.8060510754585266,
      "logps/chosen": -228.21420288085938,
      "logps/rejected": -223.0884246826172,
      "loss": 0.6856,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.01750522293150425,
      "rewards/margins": 0.016751740127801895,
      "rewards/margins_max": 0.026506105437874794,
      "rewards/margins_min": 0.006997367832809687,
      "rewards/margins_std": 0.013794762082397938,
      "rewards/rejected": 0.0007534866454079747,
      "step": 150
    },
    {
      "epoch": 0.17,
      "grad_norm": 0.345703125,
      "learning_rate": 4.920800803509025e-07,
      "logits/chosen": -1.6911264657974243,
      "logits/rejected": -0.8840080499649048,
      "logps/chosen": -303.69427490234375,
      "logps/rejected": -275.5614318847656,
      "loss": 0.6842,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.022266970947384834,
      "rewards/margins": 0.02082091197371483,
      "rewards/margins_max": 0.02934589982032776,
      "rewards/margins_min": 0.012295925989747047,
      "rewards/margins_std": 0.012056154198944569,
      "rewards/rejected": 0.0014460586244240403,
      "step": 160
    },
    {
      "epoch": 0.18,
      "grad_norm": 0.3046875,
      "learning_rate": 4.895573140745967e-07,
      "logits/chosen": -1.5039719343185425,
      "logits/rejected": -1.0316977500915527,
      "logps/chosen": -345.7312316894531,
      "logps/rejected": -288.9708251953125,
      "loss": 0.6836,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02365526184439659,
      "rewards/margins": 0.02123275026679039,
      "rewards/margins_max": 0.03283926099538803,
      "rewards/margins_min": 0.009626244194805622,
      "rewards/margins_std": 0.016414081677794456,
      "rewards/rejected": 0.002422512974590063,
      "step": 170
    },
    {
      "epoch": 0.19,
      "grad_norm": 0.265625,
      "learning_rate": 4.866946677079314e-07,
      "logits/chosen": -1.5601823329925537,
      "logits/rejected": -1.0506742000579834,
      "logps/chosen": -231.99703979492188,
      "logps/rejected": -237.2670135498047,
      "loss": 0.6828,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.022809389978647232,
      "rewards/margins": 0.020736858248710632,
      "rewards/margins_max": 0.030726289376616478,
      "rewards/margins_min": 0.010747427120804787,
      "rewards/margins_std": 0.014127190224826336,
      "rewards/rejected": 0.002072530798614025,
      "step": 180
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.365234375,
      "learning_rate": 4.834962027278417e-07,
      "logits/chosen": -1.4942362308502197,
      "logits/rejected": -0.8470790982246399,
      "logps/chosen": -291.2981872558594,
      "logps/rejected": -240.0912628173828,
      "loss": 0.682,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.017627181485295296,
      "rewards/margins": 0.022342149168252945,
      "rewards/margins_max": 0.033914387226104736,
      "rewards/margins_min": 0.01076990831643343,
      "rewards/margins_std": 0.016365615651011467,
      "rewards/rejected": -0.004714967682957649,
      "step": 190
    },
    {
      "epoch": 0.22,
      "grad_norm": 0.345703125,
      "learning_rate": 4.799664570653473e-07,
      "logits/chosen": -1.4884960651397705,
      "logits/rejected": -0.7421751022338867,
      "logps/chosen": -295.2840270996094,
      "logps/rejected": -230.6145477294922,
      "loss": 0.6808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02664310857653618,
      "rewards/margins": 0.031296949833631516,
      "rewards/margins_max": 0.04525149241089821,
      "rewards/margins_min": 0.01734241284430027,
      "rewards/margins_std": 0.0197346992790699,
      "rewards/rejected": -0.00465384079143405,
      "step": 200
    },
    {
      "epoch": 0.23,
      "grad_norm": 0.2578125,
      "learning_rate": 4.7611043866720737e-07,
      "logits/chosen": -1.49364173412323,
      "logits/rejected": -1.031049132347107,
      "logps/chosen": -258.94512939453125,
      "logps/rejected": -287.6114196777344,
      "loss": 0.6808,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.021609986200928688,
      "rewards/margins": 0.020240817219018936,
      "rewards/margins_max": 0.033506136387586594,
      "rewards/margins_min": 0.006975496653467417,
      "rewards/margins_std": 0.018759997561573982,
      "rewards/rejected": 0.0013691672356799245,
      "step": 210
    },
    {
      "epoch": 0.24,
      "grad_norm": 0.291015625,
      "learning_rate": 4.719336183907265e-07,
      "logits/chosen": -1.3044389486312866,
      "logits/rejected": -0.9332467317581177,
      "logps/chosen": -223.7823944091797,
      "logps/rejected": -217.8594207763672,
      "loss": 0.6799,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.02070688083767891,
      "rewards/margins": 0.020120607689023018,
      "rewards/margins_max": 0.03160488232970238,
      "rewards/margins_min": 0.008636328391730785,
      "rewards/margins_std": 0.016241220757365227,
      "rewards/rejected": 0.0005862751277163625,
      "step": 220
    },
    {
      "epoch": 0.25,
      "grad_norm": 0.3125,
      "learning_rate": 4.6744192224178984e-07,
      "logits/chosen": -1.3842111825942993,
      "logits/rejected": -0.9888660311698914,
      "logps/chosen": -236.2473907470703,
      "logps/rejected": -271.76641845703125,
      "loss": 0.6802,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.022707760334014893,
      "rewards/margins": 0.02468196675181389,
      "rewards/margins_max": 0.03745966777205467,
      "rewards/margins_min": 0.011904269456863403,
      "rewards/margins_std": 0.018070396035909653,
      "rewards/rejected": -0.0019742068834602833,
      "step": 230
    },
    {
      "epoch": 0.26,
      "grad_norm": 0.30078125,
      "learning_rate": 4.6264172296714e-07,
      "logits/chosen": -1.468925952911377,
      "logits/rejected": -0.8928337097167969,
      "logps/chosen": -218.7871551513672,
      "logps/rejected": -232.1916961669922,
      "loss": 0.6782,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.029307205229997635,
      "rewards/margins": 0.03086397610604763,
      "rewards/margins_max": 0.0454208180308342,
      "rewards/margins_min": 0.016307134181261063,
      "rewards/margins_std": 0.02058648318052292,
      "rewards/rejected": -0.0015567743685096502,
      "step": 240
    },
    {
      "epoch": 0.27,
      "grad_norm": 0.263671875,
      "learning_rate": 4.575398310128262e-07,
      "logits/chosen": -1.52159583568573,
      "logits/rejected": -1.062409520149231,
      "logps/chosen": -205.5279083251953,
      "logps/rejected": -209.3750457763672,
      "loss": 0.6784,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.0256162341684103,
      "rewards/margins": 0.029211264103651047,
      "rewards/margins_max": 0.04594603180885315,
      "rewards/margins_min": 0.012476496398448944,
      "rewards/margins_std": 0.023666534572839737,
      "rewards/rejected": -0.0035950313322246075,
      "step": 250
    },
    {
      "epoch": 0.28,
      "grad_norm": 0.28515625,
      "learning_rate": 4.5214348486165227e-07,
      "logits/chosen": -1.4600447416305542,
      "logits/rejected": -1.0638010501861572,
      "logps/chosen": -263.8650817871094,
      "logps/rejected": -257.2422790527344,
      "loss": 0.6772,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.031198721379041672,
      "rewards/margins": 0.032171688973903656,
      "rewards/margins_max": 0.04676546901464462,
      "rewards/margins_min": 0.01757790893316269,
      "rewards/margins_std": 0.02063872292637825,
      "rewards/rejected": -0.0009729691664688289,
      "step": 260
    },
    {
      "epoch": 0.29,
      "grad_norm": 0.333984375,
      "learning_rate": 4.4646034076333254e-07,
      "logits/chosen": -1.4118484258651733,
      "logits/rejected": -1.0390139818191528,
      "logps/chosen": -254.69589233398438,
      "logps/rejected": -277.36029052734375,
      "loss": 0.6747,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.03478916734457016,
      "rewards/margins": 0.03131258860230446,
      "rewards/margins_max": 0.0460633859038353,
      "rewards/margins_min": 0.016561787575483322,
      "rewards/margins_std": 0.020860780030488968,
      "rewards/rejected": 0.0034765794407576323,
      "step": 270
    },
    {
      "epoch": 0.3,
      "grad_norm": 0.337890625,
      "learning_rate": 4.404984618719274e-07,
      "logits/chosen": -1.5356388092041016,
      "logits/rejected": -0.8826289176940918,
      "logps/chosen": -213.24365234375,
      "logps/rejected": -205.28201293945312,
      "loss": 0.674,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.03563285619020462,
      "rewards/margins": 0.044356830418109894,
      "rewards/margins_max": 0.05965030938386917,
      "rewards/margins_min": 0.029063349589705467,
      "rewards/margins_std": 0.0216282457113266,
      "rewards/rejected": -0.008723974227905273,
      "step": 280
    },
    {
      "epoch": 0.31,
      "grad_norm": 0.322265625,
      "learning_rate": 4.342663068059689e-07,
      "logits/chosen": -1.5109997987747192,
      "logits/rejected": -1.0301908254623413,
      "logps/chosen": -227.60043334960938,
      "logps/rejected": -225.4455108642578,
      "loss": 0.6737,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.033510081470012665,
      "rewards/margins": 0.03692306950688362,
      "rewards/margins_max": 0.05596238374710083,
      "rewards/margins_min": 0.017883744090795517,
      "rewards/margins_std": 0.026925668120384216,
      "rewards/rejected": -0.0034129873383790255,
      "step": 290
    },
    {
      "epoch": 0.32,
      "grad_norm": 0.361328125,
      "learning_rate": 4.27772717647508e-07,
      "logits/chosen": -1.4661680459976196,
      "logits/rejected": -1.0104472637176514,
      "logps/chosen": -242.0974578857422,
      "logps/rejected": -232.9008026123047,
      "loss": 0.6744,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.030629118904471397,
      "rewards/margins": 0.03372279554605484,
      "rewards/margins_max": 0.050439924001693726,
      "rewards/margins_min": 0.017005670815706253,
      "rewards/margins_std": 0.023641586303710938,
      "rewards/rejected": -0.00309367710724473,
      "step": 300
    },
    {
      "epoch": 0.33,
      "grad_norm": 0.294921875,
      "learning_rate": 4.2102690739710975e-07,
      "logits/chosen": -1.35811448097229,
      "logits/rejected": -0.8667371869087219,
      "logps/chosen": -207.32943725585938,
      "logps/rejected": -239.2294464111328,
      "loss": 0.67,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.04066639393568039,
      "rewards/margins": 0.040916211903095245,
      "rewards/margins_max": 0.05701497197151184,
      "rewards/margins_min": 0.02481745555996895,
      "rewards/margins_std": 0.0227670781314373,
      "rewards/rejected": -0.0002498172107152641,
      "step": 310
    },
    {
      "epoch": 0.35,
      "grad_norm": 0.302734375,
      "learning_rate": 4.140384469025954e-07,
      "logits/chosen": -1.4642293453216553,
      "logits/rejected": -0.8783510327339172,
      "logps/chosen": -269.0649719238281,
      "logps/rejected": -253.2615966796875,
      "loss": 0.6722,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.042783185839653015,
      "rewards/margins": 0.04893990606069565,
      "rewards/margins_max": 0.07330337911844254,
      "rewards/margins_min": 0.024576421827077866,
      "rewards/margins_std": 0.03445516526699066,
      "rewards/rejected": -0.0061567178927361965,
      "step": 320
    },
    {
      "epoch": 0.36,
      "grad_norm": 0.326171875,
      "learning_rate": 4.068172512800759e-07,
      "logits/chosen": -1.4688003063201904,
      "logits/rejected": -0.9481694102287292,
      "logps/chosen": -263.0448303222656,
      "logps/rejected": -261.3179931640625,
      "loss": 0.6708,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.03902563825249672,
      "rewards/margins": 0.04815050959587097,
      "rewards/margins_max": 0.06747350096702576,
      "rewards/margins_min": 0.02882750704884529,
      "rewards/margins_std": 0.02732684649527073,
      "rewards/rejected": -0.009124869480729103,
      "step": 330
    },
    {
      "epoch": 0.37,
      "grad_norm": 0.271484375,
      "learning_rate": 3.993735658465446e-07,
      "logits/chosen": -1.3654890060424805,
      "logits/rejected": -1.0353434085845947,
      "logps/chosen": -227.32803344726562,
      "logps/rejected": -255.8052520751953,
      "loss": 0.6708,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.035697564482688904,
      "rewards/margins": 0.03952573984861374,
      "rewards/margins_max": 0.06120690703392029,
      "rewards/margins_min": 0.01784456893801689,
      "rewards/margins_std": 0.030661800876259804,
      "rewards/rejected": -0.0038281746674329042,
      "step": 340
    },
    {
      "epoch": 0.38,
      "grad_norm": 0.2734375,
      "learning_rate": 3.917179515839839e-07,
      "logits/chosen": -1.522472620010376,
      "logits/rejected": -0.8194535970687866,
      "logps/chosen": -272.1158752441406,
      "logps/rejected": -224.29052734375,
      "loss": 0.6722,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.04334510862827301,
      "rewards/margins": 0.045738715678453445,
      "rewards/margins_max": 0.07282572239637375,
      "rewards/margins_min": 0.01865171454846859,
      "rewards/margins_std": 0.03830680996179581,
      "rewards/rejected": -0.002393609844148159,
      "step": 350
    },
    {
      "epoch": 0.39,
      "grad_norm": 0.328125,
      "learning_rate": 3.8386127015561377e-07,
      "logits/chosen": -1.4874508380889893,
      "logits/rejected": -0.9508639574050903,
      "logps/chosen": -264.84063720703125,
      "logps/rejected": -285.2606506347656,
      "loss": 0.6679,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.038348861038684845,
      "rewards/margins": 0.05200430005788803,
      "rewards/margins_max": 0.07226666063070297,
      "rewards/margins_min": 0.03174193948507309,
      "rewards/margins_std": 0.028655309230089188,
      "rewards/rejected": -0.013655440881848335,
      "step": 360
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.310546875,
      "learning_rate": 3.758146684955368e-07,
      "logits/chosen": -1.4181379079818726,
      "logits/rejected": -0.9576012492179871,
      "logps/chosen": -246.5336151123047,
      "logps/rejected": -277.94232177734375,
      "loss": 0.6709,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.03755969554185867,
      "rewards/margins": 0.04255872964859009,
      "rewards/margins_max": 0.06353293359279633,
      "rewards/margins_min": 0.021584514528512955,
      "rewards/margins_std": 0.029662013053894043,
      "rewards/rejected": -0.00499903317540884,
      "step": 370
    },
    {
      "epoch": 0.41,
      "grad_norm": 0.349609375,
      "learning_rate": 3.6758956299364643e-07,
      "logits/chosen": -1.6038116216659546,
      "logits/rejected": -1.0424953699111938,
      "logps/chosen": -232.23916625976562,
      "logps/rejected": -258.908203125,
      "loss": 0.6694,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.044283345341682434,
      "rewards/margins": 0.04850899800658226,
      "rewards/margins_max": 0.07648013532161713,
      "rewards/margins_min": 0.020537864416837692,
      "rewards/margins_std": 0.039557162672281265,
      "rewards/rejected": -0.004225648939609528,
      "step": 380
    },
    {
      "epoch": 0.42,
      "grad_norm": 0.318359375,
      "learning_rate": 3.591976232982355e-07,
      "logits/chosen": -1.4831396341323853,
      "logits/rejected": -0.7363861203193665,
      "logps/chosen": -274.75311279296875,
      "logps/rejected": -217.2397918701172,
      "loss": 0.6673,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.052159328013658524,
      "rewards/margins": 0.05904749035835266,
      "rewards/margins_max": 0.08474520593881607,
      "rewards/margins_min": 0.03334975615143776,
      "rewards/margins_std": 0.0363420732319355,
      "rewards/rejected": -0.006888158619403839,
      "step": 390
    },
    {
      "epoch": 0.43,
      "grad_norm": 0.3046875,
      "learning_rate": 3.506507557592853e-07,
      "logits/chosen": -1.4179537296295166,
      "logits/rejected": -0.8838945627212524,
      "logps/chosen": -329.03863525390625,
      "logps/rejected": -292.02008056640625,
      "loss": 0.6655,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.044211916625499725,
      "rewards/margins": 0.057223010808229446,
      "rewards/margins_max": 0.08557866513729095,
      "rewards/margins_min": 0.028867345303297043,
      "rewards/margins_std": 0.04010096564888954,
      "rewards/rejected": -0.013011088594794273,
      "step": 400
    },
    {
      "epoch": 0.44,
      "grad_norm": 0.4375,
      "learning_rate": 3.419610865359266e-07,
      "logits/chosen": -1.4828202724456787,
      "logits/rejected": -0.8764745593070984,
      "logps/chosen": -280.54998779296875,
      "logps/rejected": -276.4558410644531,
      "loss": 0.6682,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.05212799459695816,
      "rewards/margins": 0.05141522362828255,
      "rewards/margins_max": 0.08034422993659973,
      "rewards/margins_min": 0.022486215457320213,
      "rewards/margins_std": 0.04091179370880127,
      "rewards/rejected": 0.0007127688149921596,
      "step": 410
    },
    {
      "epoch": 0.45,
      "grad_norm": 0.33984375,
      "learning_rate": 3.33140944392039e-07,
      "logits/chosen": -1.2407147884368896,
      "logits/rejected": -0.8896020650863647,
      "logps/chosen": -235.588134765625,
      "logps/rejected": -242.38858032226562,
      "loss": 0.6687,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.03107466734945774,
      "rewards/margins": 0.04102586954832077,
      "rewards/margins_max": 0.0646006390452385,
      "rewards/margins_min": 0.017451094463467598,
      "rewards/margins_std": 0.033339761197566986,
      "rewards/rejected": -0.00995120219886303,
      "step": 420
    },
    {
      "epoch": 0.46,
      "grad_norm": 0.3203125,
      "learning_rate": 3.2420284320439736e-07,
      "logits/chosen": -1.5982177257537842,
      "logits/rejected": -0.8901177644729614,
      "logps/chosen": -235.1398468017578,
      "logps/rejected": -225.75119018554688,
      "loss": 0.667,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.05144185945391655,
      "rewards/margins": 0.0693436712026596,
      "rewards/margins_max": 0.09611108899116516,
      "rewards/margins_min": 0.04257623478770256,
      "rewards/margins_std": 0.03785485774278641,
      "rewards/rejected": -0.01790180616080761,
      "step": 430
    },
    {
      "epoch": 0.47,
      "grad_norm": 0.291015625,
      "learning_rate": 3.151594642081834e-07,
      "logits/chosen": -1.5106613636016846,
      "logits/rejected": -0.9530634880065918,
      "logps/chosen": -259.29364013671875,
      "logps/rejected": -263.5410461425781,
      "loss": 0.6681,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.054123084992170334,
      "rewards/margins": 0.0667182207107544,
      "rewards/margins_max": 0.09307887405157089,
      "rewards/margins_min": 0.0403575673699379,
      "rewards/margins_std": 0.03727959841489792,
      "rewards/rejected": -0.012595141306519508,
      "step": 440
    },
    {
      "epoch": 0.49,
      "grad_norm": 0.341796875,
      "learning_rate": 3.060236380050519e-07,
      "logits/chosen": -1.5215215682983398,
      "logits/rejected": -0.915096640586853,
      "logps/chosen": -241.9713897705078,
      "logps/rejected": -212.6835174560547,
      "loss": 0.6648,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.041197773069143295,
      "rewards/margins": 0.06050584465265274,
      "rewards/margins_max": 0.09923966228961945,
      "rewards/margins_min": 0.021772030740976334,
      "rewards/margins_std": 0.05477788299322128,
      "rewards/rejected": -0.019308075308799744,
      "step": 450
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.369140625,
      "learning_rate": 2.968083263592782e-07,
      "logits/chosen": -1.429099202156067,
      "logits/rejected": -0.9603363275527954,
      "logps/chosen": -226.94985961914062,
      "logps/rejected": -231.18212890625,
      "loss": 0.6669,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.039319347590208054,
      "rewards/margins": 0.0540069118142128,
      "rewards/margins_max": 0.08096911013126373,
      "rewards/margins_min": 0.027044707909226418,
      "rewards/margins_std": 0.03813030570745468,
      "rewards/rejected": -0.014687557704746723,
      "step": 460
    },
    {
      "epoch": 0.51,
      "grad_norm": 0.3125,
      "learning_rate": 2.875266038078136e-07,
      "logits/chosen": -1.467827320098877,
      "logits/rejected": -0.8274309039115906,
      "logps/chosen": -262.02984619140625,
      "logps/rejected": -258.05291748046875,
      "loss": 0.6663,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.047955792397260666,
      "rewards/margins": 0.05926694720983505,
      "rewards/margins_max": 0.08313200622797012,
      "rewards/margins_min": 0.035401880741119385,
      "rewards/margins_std": 0.033750299364328384,
      "rewards/rejected": -0.011311152949929237,
      "step": 470
    },
    {
      "epoch": 0.52,
      "grad_norm": 0.330078125,
      "learning_rate": 2.781916391103417e-07,
      "logits/chosen": -1.4126121997833252,
      "logits/rejected": -1.062652826309204,
      "logps/chosen": -312.4531555175781,
      "logps/rejected": -325.8970642089844,
      "loss": 0.6669,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.046898700296878815,
      "rewards/margins": 0.05282552167773247,
      "rewards/margins_max": 0.07998780906200409,
      "rewards/margins_min": 0.025663232430815697,
      "rewards/margins_std": 0.03841327875852585,
      "rewards/rejected": -0.005926821380853653,
      "step": 480
    },
    {
      "epoch": 0.53,
      "grad_norm": 0.3359375,
      "learning_rate": 2.6881667656565226e-07,
      "logits/chosen": -1.4687728881835938,
      "logits/rejected": -0.979697048664093,
      "logps/chosen": -241.0842742919922,
      "logps/rejected": -232.9574432373047,
      "loss": 0.6659,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.054931215941905975,
      "rewards/margins": 0.0482785627245903,
      "rewards/margins_max": 0.0750807598233223,
      "rewards/margins_min": 0.02147636190056801,
      "rewards/margins_std": 0.037904031574726105,
      "rewards/rejected": 0.006652662996202707,
      "step": 490
    },
    {
      "epoch": 0.54,
      "grad_norm": 0.28515625,
      "learning_rate": 2.594150172208416e-07,
      "logits/chosen": -1.5185790061950684,
      "logits/rejected": -0.9422761797904968,
      "logps/chosen": -234.38119506835938,
      "logps/rejected": -252.7394561767578,
      "loss": 0.669,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.04691288247704506,
      "rewards/margins": 0.05338384583592415,
      "rewards/margins_max": 0.08102253079414368,
      "rewards/margins_min": 0.025745173916220665,
      "rewards/margins_std": 0.03908699378371239,
      "rewards/rejected": -0.006470963358879089,
      "step": 500
    },
    {
      "epoch": 0.55,
      "grad_norm": 0.302734375,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.4481227397918701,
      "logits/rejected": -1.0346171855926514,
      "logps/chosen": -220.05325317382812,
      "logps/rejected": -236.8842315673828,
      "loss": 0.6634,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.044769201427698135,
      "rewards/margins": 0.05242365598678589,
      "rewards/margins_max": 0.08209005743265152,
      "rewards/margins_min": 0.022757260128855705,
      "rewards/margins_std": 0.04195461794734001,
      "rewards/rejected": -0.007654457353055477,
      "step": 510
    },
    {
      "epoch": 0.56,
      "grad_norm": 0.31640625,
      "learning_rate": 2.405849827791583e-07,
      "logits/chosen": -1.3833669424057007,
      "logits/rejected": -0.881574273109436,
      "logps/chosen": -241.27481079101562,
      "logps/rejected": -263.91351318359375,
      "loss": 0.6641,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05363880842924118,
      "rewards/margins": 0.065273717045784,
      "rewards/margins_max": 0.09218905121088028,
      "rewards/margins_min": 0.03835836052894592,
      "rewards/margins_std": 0.03806404396891594,
      "rewards/rejected": -0.011634895578026772,
      "step": 520
    },
    {
      "epoch": 0.57,
      "grad_norm": 0.3671875,
      "learning_rate": 2.3118332343434777e-07,
      "logits/chosen": -1.5236294269561768,
      "logits/rejected": -0.975500762462616,
      "logps/chosen": -249.5443572998047,
      "logps/rejected": -252.99618530273438,
      "loss": 0.6649,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.057201676070690155,
      "rewards/margins": 0.06602860987186432,
      "rewards/margins_max": 0.08838556706905365,
      "rewards/margins_min": 0.04367166385054588,
      "rewards/margins_std": 0.03161751106381416,
      "rewards/rejected": -0.00882694311439991,
      "step": 530
    },
    {
      "epoch": 0.58,
      "grad_norm": 0.2734375,
      "learning_rate": 2.218083608896583e-07,
      "logits/chosen": -1.4163635969161987,
      "logits/rejected": -1.021444320678711,
      "logps/chosen": -238.6166534423828,
      "logps/rejected": -231.279296875,
      "loss": 0.6678,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.04542135074734688,
      "rewards/margins": 0.04465199261903763,
      "rewards/margins_max": 0.07032604515552521,
      "rewards/margins_min": 0.018977930769324303,
      "rewards/margins_std": 0.03630860149860382,
      "rewards/rejected": 0.000769357371609658,
      "step": 540
    },
    {
      "epoch": 0.59,
      "grad_norm": 0.298828125,
      "learning_rate": 2.1247339619218638e-07,
      "logits/chosen": -1.5570924282073975,
      "logits/rejected": -0.9521455764770508,
      "logps/chosen": -244.45877075195312,
      "logps/rejected": -218.03067016601562,
      "loss": 0.6633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04918726533651352,
      "rewards/margins": 0.05797078087925911,
      "rewards/margins_max": 0.07724090665578842,
      "rewards/margins_min": 0.0387006476521492,
      "rewards/margins_std": 0.027252081781625748,
      "rewards/rejected": -0.008783518336713314,
      "step": 550
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.369140625,
      "learning_rate": 2.031916736407218e-07,
      "logits/chosen": -1.246797800064087,
      "logits/rejected": -0.8754084706306458,
      "logps/chosen": -255.4636688232422,
      "logps/rejected": -206.0786590576172,
      "loss": 0.6682,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.04116806760430336,
      "rewards/margins": 0.05079611390829086,
      "rewards/margins_max": 0.07820285856723785,
      "rewards/margins_min": 0.023389369249343872,
      "rewards/margins_std": 0.03875899314880371,
      "rewards/rejected": -0.009628048166632652,
      "step": 560
    },
    {
      "epoch": 0.61,
      "grad_norm": 0.3125,
      "learning_rate": 1.9397636199494806e-07,
      "logits/chosen": -1.3417741060256958,
      "logits/rejected": -0.9931136965751648,
      "logps/chosen": -245.11184692382812,
      "logps/rejected": -274.5023498535156,
      "loss": 0.667,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.04023710638284683,
      "rewards/margins": 0.060276590287685394,
      "rewards/margins_max": 0.08814635127782822,
      "rewards/margins_min": 0.03240684047341347,
      "rewards/margins_std": 0.03941378742456436,
      "rewards/rejected": -0.02003948949277401,
      "step": 570
    },
    {
      "epoch": 0.63,
      "grad_norm": 0.310546875,
      "learning_rate": 1.8484053579181658e-07,
      "logits/chosen": -1.4128963947296143,
      "logits/rejected": -0.9094281196594238,
      "logps/chosen": -241.9584503173828,
      "logps/rejected": -255.27676391601562,
      "loss": 0.6669,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.04914793372154236,
      "rewards/margins": 0.06206917762756348,
      "rewards/margins_max": 0.09373507648706436,
      "rewards/margins_min": 0.030403289943933487,
      "rewards/margins_std": 0.044782333076000214,
      "rewards/rejected": -0.012921245768666267,
      "step": 580
    },
    {
      "epoch": 0.64,
      "grad_norm": 0.3828125,
      "learning_rate": 1.757971567956027e-07,
      "logits/chosen": -1.7144603729248047,
      "logits/rejected": -0.9307141304016113,
      "logps/chosen": -272.17547607421875,
      "logps/rejected": -241.65670776367188,
      "loss": 0.6637,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.05295403674244881,
      "rewards/margins": 0.05789683386683464,
      "rewards/margins_max": 0.08770729601383209,
      "rewards/margins_min": 0.028086364269256592,
      "rewards/margins_std": 0.04215836524963379,
      "rewards/rejected": -0.00494279433041811,
      "step": 590
    },
    {
      "epoch": 0.65,
      "grad_norm": 0.2890625,
      "learning_rate": 1.6685905560796098e-07,
      "logits/chosen": -1.3933067321777344,
      "logits/rejected": -0.8825523257255554,
      "logps/chosen": -218.8330078125,
      "logps/rejected": -243.945556640625,
      "loss": 0.6669,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.044868241995573044,
      "rewards/margins": 0.04994089528918266,
      "rewards/margins_max": 0.07978564500808716,
      "rewards/margins_min": 0.020096149295568466,
      "rewards/margins_std": 0.04220684990286827,
      "rewards/rejected": -0.005072650499641895,
      "step": 600
    },
    {
      "epoch": 0.66,
      "grad_norm": 0.384765625,
      "learning_rate": 1.580389134640734e-07,
      "logits/chosen": -1.4454293251037598,
      "logits/rejected": -1.0624114274978638,
      "logps/chosen": -232.04562377929688,
      "logps/rejected": -241.92013549804688,
      "loss": 0.6626,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.048992957919836044,
      "rewards/margins": 0.06253460794687271,
      "rewards/margins_max": 0.09021260589361191,
      "rewards/margins_min": 0.034856610000133514,
      "rewards/margins_std": 0.03914260491728783,
      "rewards/rejected": -0.013541650958359241,
      "step": 610
    },
    {
      "epoch": 0.67,
      "grad_norm": 0.41796875,
      "learning_rate": 1.4934924424071475e-07,
      "logits/chosen": -1.5101556777954102,
      "logits/rejected": -0.8701263666152954,
      "logps/chosen": -268.46563720703125,
      "logps/rejected": -246.90725708007812,
      "loss": 0.6673,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.047949280589818954,
      "rewards/margins": 0.05510631948709488,
      "rewards/margins_max": 0.08622908592224121,
      "rewards/margins_min": 0.023983558639883995,
      "rewards/margins_std": 0.044014234095811844,
      "rewards/rejected": -0.00715703796595335,
      "step": 620
    },
    {
      "epoch": 0.68,
      "grad_norm": 0.345703125,
      "learning_rate": 1.4080237670176453e-07,
      "logits/chosen": -1.5899397134780884,
      "logits/rejected": -0.9542080163955688,
      "logps/chosen": -250.47073364257812,
      "logps/rejected": -213.67672729492188,
      "loss": 0.6642,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.048125170171260834,
      "rewards/margins": 0.058349937200546265,
      "rewards/margins_max": 0.08438356220722198,
      "rewards/margins_min": 0.03231631591916084,
      "rewards/margins_std": 0.03681711107492447,
      "rewards/rejected": -0.010224771685898304,
      "step": 630
    },
    {
      "epoch": 0.69,
      "grad_norm": 0.32421875,
      "learning_rate": 1.3241043700635352e-07,
      "logits/chosen": -1.4617611169815063,
      "logits/rejected": -0.715996265411377,
      "logps/chosen": -311.3377990722656,
      "logps/rejected": -235.23257446289062,
      "loss": 0.6615,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.049124039709568024,
      "rewards/margins": 0.06973692029714584,
      "rewards/margins_max": 0.09559544920921326,
      "rewards/margins_min": 0.04387838765978813,
      "rewards/margins_std": 0.03656948357820511,
      "rewards/rejected": -0.02061288245022297,
      "step": 640
    },
    {
      "epoch": 0.7,
      "grad_norm": 0.341796875,
      "learning_rate": 1.2418533150446324e-07,
      "logits/chosen": -1.5678064823150635,
      "logits/rejected": -0.8574711680412292,
      "logps/chosen": -270.28997802734375,
      "logps/rejected": -229.6758575439453,
      "loss": 0.6643,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.049732841551303864,
      "rewards/margins": 0.06726487725973129,
      "rewards/margins_max": 0.09402552247047424,
      "rewards/margins_min": 0.04050421714782715,
      "rewards/margins_std": 0.03784528002142906,
      "rewards/rejected": -0.01753203384578228,
      "step": 650
    },
    {
      "epoch": 0.71,
      "grad_norm": 0.322265625,
      "learning_rate": 1.1613872984438628e-07,
      "logits/chosen": -1.5800104141235352,
      "logits/rejected": -0.9640630483627319,
      "logps/chosen": -217.96762084960938,
      "logps/rejected": -209.03237915039062,
      "loss": 0.6664,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.05414942651987076,
      "rewards/margins": 0.05594904348254204,
      "rewards/margins_max": 0.08529958873987198,
      "rewards/margins_min": 0.026598507538437843,
      "rewards/margins_std": 0.04150792956352234,
      "rewards/rejected": -0.0017996244132518768,
      "step": 660
    },
    {
      "epoch": 0.72,
      "grad_norm": 0.412109375,
      "learning_rate": 1.0828204841601607e-07,
      "logits/chosen": -1.573081612586975,
      "logits/rejected": -1.0643428564071655,
      "logps/chosen": -267.5829772949219,
      "logps/rejected": -279.0192565917969,
      "loss": 0.6654,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05038607865571976,
      "rewards/margins": 0.059136874973773956,
      "rewards/margins_max": 0.08540179580450058,
      "rewards/margins_min": 0.032871946692466736,
      "rewards/margins_std": 0.037144217640161514,
      "rewards/rejected": -0.00875079445540905,
      "step": 670
    },
    {
      "epoch": 0.73,
      "grad_norm": 0.3046875,
      "learning_rate": 1.0062643415345545e-07,
      "logits/chosen": -1.5550715923309326,
      "logits/rejected": -0.911180853843689,
      "logps/chosen": -234.7667999267578,
      "logps/rejected": -255.4190673828125,
      "loss": 0.6641,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.051492560654878616,
      "rewards/margins": 0.06578966230154037,
      "rewards/margins_max": 0.09979908168315887,
      "rewards/margins_min": 0.03178024664521217,
      "rewards/margins_std": 0.04809657856822014,
      "rewards/rejected": -0.014297107234597206,
      "step": 680
    },
    {
      "epoch": 0.74,
      "grad_norm": 0.287109375,
      "learning_rate": 9.318274871992407e-08,
      "logits/chosen": -1.5272128582000732,
      "logits/rejected": -1.013564944267273,
      "logps/chosen": -241.36782836914062,
      "logps/rejected": -224.7821044921875,
      "loss": 0.6665,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.04677369445562363,
      "rewards/margins": 0.05521191284060478,
      "rewards/margins_max": 0.08911783993244171,
      "rewards/margins_min": 0.021305980160832405,
      "rewards/margins_std": 0.04795023053884506,
      "rewards/rejected": -0.008438214659690857,
      "step": 690
    },
    {
      "epoch": 0.76,
      "grad_norm": 0.314453125,
      "learning_rate": 8.596155309740469e-08,
      "logits/chosen": -1.6033340692520142,
      "logits/rejected": -1.09574294090271,
      "logps/chosen": -246.16799926757812,
      "logps/rejected": -263.65142822265625,
      "loss": 0.6648,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0439617782831192,
      "rewards/margins": 0.054900698363780975,
      "rewards/margins_max": 0.08236662298440933,
      "rewards/margins_min": 0.02743479050695896,
      "rewards/margins_std": 0.03884267061948776,
      "rewards/rejected": -0.010938925668597221,
      "step": 700
    },
    {
      "epoch": 0.77,
      "grad_norm": 0.267578125,
      "learning_rate": 7.897309260289026e-08,
      "logits/chosen": -1.5482122898101807,
      "logits/rejected": -1.1510074138641357,
      "logps/chosen": -249.0963897705078,
      "logps/rejected": -259.163818359375,
      "loss": 0.6635,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.04723275452852249,
      "rewards/margins": 0.05302921682596207,
      "rewards/margins_max": 0.0721876472234726,
      "rewards/margins_min": 0.03387077525258064,
      "rewards/margins_std": 0.027094120159745216,
      "rewards/rejected": -0.005796459037810564,
      "step": 710
    },
    {
      "epoch": 0.78,
      "grad_norm": 0.373046875,
      "learning_rate": 7.222728235249195e-08,
      "logits/chosen": -1.3384692668914795,
      "logits/rejected": -0.7518659830093384,
      "logps/chosen": -202.97393798828125,
      "logps/rejected": -189.1539764404297,
      "loss": 0.6669,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.03645472601056099,
      "rewards/margins": 0.05072823911905289,
      "rewards/margins_max": 0.0754196047782898,
      "rewards/margins_min": 0.026036862283945084,
      "rewards/margins_std": 0.03491886705160141,
      "rewards/rejected": -0.014273506589233875,
      "step": 720
    },
    {
      "epoch": 0.79,
      "grad_norm": 0.28125,
      "learning_rate": 6.573369319403108e-08,
      "logits/chosen": -1.5845191478729248,
      "logits/rejected": -0.9599083065986633,
      "logps/chosen": -228.37417602539062,
      "logps/rejected": -237.9397430419922,
      "loss": 0.6648,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.05403770133852959,
      "rewards/margins": 0.06774094700813293,
      "rewards/margins_max": 0.09628431499004364,
      "rewards/margins_min": 0.03919757157564163,
      "rewards/margins_std": 0.04036641865968704,
      "rewards/rejected": -0.013703237287700176,
      "step": 730
    },
    {
      "epoch": 0.8,
      "grad_norm": 0.357421875,
      "learning_rate": 5.9501538128072597e-08,
      "logits/chosen": -1.5676336288452148,
      "logits/rejected": -0.8547343015670776,
      "logps/chosen": -290.7181701660156,
      "logps/rejected": -234.4829559326172,
      "loss": 0.6666,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.046895284205675125,
      "rewards/margins": 0.059913188219070435,
      "rewards/margins_max": 0.08948854357004166,
      "rewards/margins_min": 0.030337834730744362,
      "rewards/margins_std": 0.04182586818933487,
      "rewards/rejected": -0.01301790215075016,
      "step": 740
    },
    {
      "epoch": 0.81,
      "grad_norm": 0.34765625,
      "learning_rate": 5.353965923666742e-08,
      "logits/chosen": -1.3945667743682861,
      "logits/rejected": -0.8627855181694031,
      "logps/chosen": -309.19976806640625,
      "logps/rejected": -313.39263916015625,
      "loss": 0.6653,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.052454281598329544,
      "rewards/margins": 0.06173131614923477,
      "rewards/margins_max": 0.08643205463886261,
      "rewards/margins_min": 0.037030577659606934,
      "rewards/margins_std": 0.03493211418390274,
      "rewards/rejected": -0.009277036413550377,
      "step": 750
    },
    {
      "epoch": 0.82,
      "grad_norm": 0.3046875,
      "learning_rate": 4.7856515138347735e-08,
      "logits/chosen": -1.501372218132019,
      "logits/rejected": -0.781902015209198,
      "logps/chosen": -265.1309509277344,
      "logps/rejected": -230.6505126953125,
      "loss": 0.6627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05485969036817551,
      "rewards/margins": 0.0646880641579628,
      "rewards/margins_max": 0.08704294264316559,
      "rewards/margins_min": 0.04233316332101822,
      "rewards/margins_std": 0.03161459416151047,
      "rewards/rejected": -0.009828361682593822,
      "step": 760
    },
    {
      "epoch": 0.83,
      "grad_norm": 0.359375,
      "learning_rate": 4.2460168987173806e-08,
      "logits/chosen": -1.595336675643921,
      "logits/rejected": -0.9258917570114136,
      "logps/chosen": -295.4404602050781,
      "logps/rejected": -247.3825225830078,
      "loss": 0.6617,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.056455206125974655,
      "rewards/margins": 0.0680319294333458,
      "rewards/margins_max": 0.09642402827739716,
      "rewards/margins_min": 0.039639830589294434,
      "rewards/margins_std": 0.040152497589588165,
      "rewards/rejected": -0.011576727032661438,
      "step": 770
    },
    {
      "epoch": 0.84,
      "grad_norm": 0.326171875,
      "learning_rate": 3.7358277032860016e-08,
      "logits/chosen": -1.5608649253845215,
      "logits/rejected": -0.8827853202819824,
      "logps/chosen": -260.73944091796875,
      "logps/rejected": -277.09674072265625,
      "loss": 0.6638,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.049758292734622955,
      "rewards/margins": 0.06226016953587532,
      "rewards/margins_max": 0.08762570470571518,
      "rewards/margins_min": 0.03689463064074516,
      "rewards/margins_std": 0.035872288048267365,
      "rewards/rejected": -0.012501873075962067,
      "step": 780
    },
    {
      "epoch": 0.85,
      "grad_norm": 0.251953125,
      "learning_rate": 3.255807775821015e-08,
      "logits/chosen": -1.5679352283477783,
      "logits/rejected": -0.9218677282333374,
      "logps/chosen": -289.8990783691406,
      "logps/rejected": -239.71261596679688,
      "loss": 0.6657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05259973928332329,
      "rewards/margins": 0.06116511672735214,
      "rewards/margins_max": 0.08521325886249542,
      "rewards/margins_min": 0.037116967141628265,
      "rewards/margins_std": 0.034009214490652084,
      "rewards/rejected": -0.00856537651270628,
      "step": 790
    },
    {
      "epoch": 0.86,
      "grad_norm": 0.306640625,
      "learning_rate": 2.8066381609273493e-08,
      "logits/chosen": -1.4278221130371094,
      "logits/rejected": -0.8260752558708191,
      "logps/chosen": -247.1492156982422,
      "logps/rejected": -225.48062133789062,
      "loss": 0.6659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05115921422839165,
      "rewards/margins": 0.0625448077917099,
      "rewards/margins_max": 0.09311069548130035,
      "rewards/margins_min": 0.031978923827409744,
      "rewards/margins_std": 0.043226685374975204,
      "rewards/rejected": -0.011385595425963402,
      "step": 800
    },
    {
      "epoch": 0.87,
      "grad_norm": 0.3359375,
      "learning_rate": 2.3889561332792657e-08,
      "logits/chosen": -1.5297521352767944,
      "logits/rejected": -0.9847742319107056,
      "logps/chosen": -270.2022399902344,
      "logps/rejected": -246.3189239501953,
      "loss": 0.6656,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.04832616075873375,
      "rewards/margins": 0.057206034660339355,
      "rewards/margins_max": 0.08940082043409348,
      "rewards/margins_min": 0.025011247023940086,
      "rewards/margins_std": 0.04553030803799629,
      "rewards/rejected": -0.008879872970283031,
      "step": 810
    },
    {
      "epoch": 0.88,
      "grad_norm": 0.3203125,
      "learning_rate": 2.0033542934652675e-08,
      "logits/chosen": -1.5510307550430298,
      "logits/rejected": -0.856239914894104,
      "logps/chosen": -256.7835388183594,
      "logps/rejected": -223.23135375976562,
      "loss": 0.6654,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.052466489374637604,
      "rewards/margins": 0.058548521250486374,
      "rewards/margins_max": 0.08875375986099243,
      "rewards/margins_min": 0.028343280777335167,
      "rewards/margins_std": 0.042716652154922485,
      "rewards/rejected": -0.006082023028284311,
      "step": 820
    },
    {
      "epoch": 0.9,
      "grad_norm": 0.333984375,
      "learning_rate": 1.6503797272158282e-08,
      "logits/chosen": -1.4320178031921387,
      "logits/rejected": -1.0073983669281006,
      "logps/chosen": -241.7576904296875,
      "logps/rejected": -277.2112731933594,
      "loss": 0.6654,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.03875169903039932,
      "rewards/margins": 0.05952931568026543,
      "rewards/margins_max": 0.09110890328884125,
      "rewards/margins_min": 0.027949709445238113,
      "rewards/margins_std": 0.044660307466983795,
      "rewards/rejected": -0.020777616649866104,
      "step": 830
    },
    {
      "epoch": 0.91,
      "grad_norm": 0.291015625,
      "learning_rate": 1.3305332292068705e-08,
      "logits/chosen": -1.5706042051315308,
      "logits/rejected": -1.0899040699005127,
      "logps/chosen": -276.74017333984375,
      "logps/rejected": -287.7333068847656,
      "loss": 0.6645,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.05088504031300545,
      "rewards/margins": 0.0576903410255909,
      "rewards/margins_max": 0.0889071449637413,
      "rewards/margins_min": 0.02647354081273079,
      "rewards/margins_std": 0.04414721950888634,
      "rewards/rejected": -0.006805300712585449,
      "step": 840
    },
    {
      "epoch": 0.92,
      "grad_norm": 0.33203125,
      "learning_rate": 1.0442685925403344e-08,
      "logits/chosen": -1.5475889444351196,
      "logits/rejected": -1.067118525505066,
      "logps/chosen": -247.9560089111328,
      "logps/rejected": -261.21209716796875,
      "loss": 0.6645,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.054555535316467285,
      "rewards/margins": 0.06061048060655594,
      "rewards/margins_max": 0.09440271556377411,
      "rewards/margins_min": 0.026818236336112022,
      "rewards/margins_std": 0.04778943955898285,
      "rewards/rejected": -0.00605494249612093,
      "step": 850
    },
    {
      "epoch": 0.93,
      "grad_norm": 0.35546875,
      "learning_rate": 7.91991964909744e-09,
      "logits/chosen": -1.5974628925323486,
      "logits/rejected": -0.9376012682914734,
      "logps/chosen": -226.9661407470703,
      "logps/rejected": -215.2276153564453,
      "loss": 0.6649,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.04701418802142143,
      "rewards/margins": 0.06498473882675171,
      "rewards/margins_max": 0.08930108696222305,
      "rewards/margins_min": 0.040668390691280365,
      "rewards/margins_std": 0.034388504922389984,
      "rewards/rejected": -0.017970550805330276,
      "step": 860
    },
    {
      "epoch": 0.94,
      "grad_norm": 0.259765625,
      "learning_rate": 5.740612723643401e-09,
      "logits/chosen": -1.5247188806533813,
      "logits/rejected": -1.0012189149856567,
      "logps/chosen": -215.85086059570312,
      "logps/rejected": -213.22756958007812,
      "loss": 0.6651,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.03825182095170021,
      "rewards/margins": 0.05696084350347519,
      "rewards/margins_max": 0.08913502097129822,
      "rewards/margins_min": 0.024786660447716713,
      "rewards/margins_std": 0.045501161366701126,
      "rewards/rejected": -0.01870902255177498,
      "step": 870
    },
    {
      "epoch": 0.95,
      "grad_norm": 0.3203125,
      "learning_rate": 3.907857114893359e-09,
      "logits/chosen": -1.526238203048706,
      "logits/rejected": -1.0228248834609985,
      "logps/chosen": -250.3122100830078,
      "logps/rejected": -268.1669616699219,
      "loss": 0.6645,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.05541209131479263,
      "rewards/margins": 0.06139000505208969,
      "rewards/margins_max": 0.08599219471216202,
      "rewards/margins_min": 0.03678782656788826,
      "rewards/margins_std": 0.03479274362325668,
      "rewards/rejected": -0.005977921187877655,
      "step": 880
    },
    {
      "epoch": 0.96,
      "grad_norm": 0.390625,
      "learning_rate": 2.4242531072273255e-09,
      "logits/chosen": -1.6231883764266968,
      "logits/rejected": -1.0414937734603882,
      "logps/chosen": -242.6604766845703,
      "logps/rejected": -248.1051483154297,
      "loss": 0.6626,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.04938145726919174,
      "rewards/margins": 0.05951399728655815,
      "rewards/margins_max": 0.08650043606758118,
      "rewards/margins_min": 0.03252756968140602,
      "rewards/margins_std": 0.03816457465291023,
      "rewards/rejected": -0.01013254001736641,
      "step": 890
    },
    {
      "epoch": 0.97,
      "grad_norm": 0.279296875,
      "learning_rate": 1.2919056143113061e-09,
      "logits/chosen": -1.6871249675750732,
      "logits/rejected": -1.0585591793060303,
      "logps/chosen": -217.6454620361328,
      "logps/rejected": -223.8054656982422,
      "loss": 0.6662,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.048433009535074234,
      "rewards/margins": 0.06717512011528015,
      "rewards/margins_max": 0.09603999555110931,
      "rewards/margins_min": 0.03831023350358009,
      "rewards/margins_std": 0.04082110896706581,
      "rewards/rejected": -0.01874210312962532,
      "step": 900
    },
    {
      "epoch": 0.98,
      "grad_norm": 0.326171875,
      "learning_rate": 5.124211926793575e-10,
      "logits/chosen": -1.413845419883728,
      "logits/rejected": -0.7665145993232727,
      "logps/chosen": -264.106689453125,
      "logps/rejected": -229.184326171875,
      "loss": 0.6668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0459374263882637,
      "rewards/margins": 0.05880744382739067,
      "rewards/margins_max": 0.08354458957910538,
      "rewards/margins_min": 0.034070298075675964,
      "rewards/margins_std": 0.03498360887169838,
      "rewards/rejected": -0.012870019301772118,
      "step": 910
    },
    {
      "epoch": 0.99,
      "grad_norm": 0.341796875,
      "learning_rate": 8.690576237688207e-11,
      "logits/chosen": -1.4011876583099365,
      "logits/rejected": -0.9543458819389343,
      "logps/chosen": -281.8406677246094,
      "logps/rejected": -222.5669708251953,
      "loss": 0.6665,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.045102305710315704,
      "rewards/margins": 0.05106315761804581,
      "rewards/margins_max": 0.07114427536725998,
      "rewards/margins_min": 0.03098202869296074,
      "rewards/margins_std": 0.028398994356393814,
      "rewards/rejected": -0.0059608458541333675,
      "step": 920
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -1.0221611261367798,
      "eval_logits/rejected": -0.8962497115135193,
      "eval_logps/chosen": -331.233642578125,
      "eval_logps/rejected": -327.55169677734375,
      "eval_loss": 0.6933034658432007,
      "eval_rewards/accuracies": 0.5360000133514404,
      "eval_rewards/chosen": 0.026766540482640266,
      "eval_rewards/margins": 0.0002464489371050149,
      "eval_rewards/margins_max": 0.06006291136145592,
      "eval_rewards/margins_min": -0.061876267194747925,
      "eval_rewards/margins_std": 0.04059867188334465,
      "eval_rewards/rejected": 0.02652009204030037,
      "eval_runtime": 750.1106,
      "eval_samples_per_second": 5.333,
      "eval_steps_per_second": 0.167,
      "step": 927
    },
    {
      "epoch": 1.0,
      "step": 927,
      "total_flos": 0.0,
      "train_loss": 0.6720605961327414,
      "train_runtime": 8514.5547,
      "train_samples_per_second": 1.741,
      "train_steps_per_second": 0.109
    }
  ],
  "logging_steps": 10,
  "max_steps": 927,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}