| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 32, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/policy_chosen_logits": 1.5414493083953857, | |
| "debug/policy_chosen_logps": -465.5361328125, | |
| "debug/policy_rejected_logits": 1.6527317762374878, | |
| "debug/policy_rejected_logps": -427.4836120605469, | |
| "debug/reference_chosen_logps": -465.5361328125, | |
| "debug/reference_rejected_logps": -427.4836120605469, | |
| "epoch": 0.03125, | |
| "grad_norm": 12.323765072380743, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.5414493083953857, | |
| "logits/rejected": 1.6527317762374878, | |
| "logps/chosen": -465.5361328125, | |
| "logps/rejected": -427.4836120605469, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.8114635944366455, | |
| "debug/policy_chosen_logps": -395.85809326171875, | |
| "debug/policy_rejected_logits": 2.0628278255462646, | |
| "debug/policy_rejected_logps": -405.449462890625, | |
| "debug/reference_chosen_logps": -395.2831726074219, | |
| "debug/reference_rejected_logps": -405.4757080078125, | |
| "epoch": 0.0625, | |
| "grad_norm": 5.688989109980878, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.8114635944366455, | |
| "logits/rejected": 2.0628278255462646, | |
| "logps/chosen": -395.85809326171875, | |
| "logps/rejected": -405.449462890625, | |
| "loss": 0.5021, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.005749321077018976, | |
| "rewards/margins": -0.006011696066707373, | |
| "rewards/rejected": 0.0002623748732730746, | |
| "step": 2 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.734413504600525, | |
| "debug/policy_chosen_logps": -416.40167236328125, | |
| "debug/policy_rejected_logits": 1.5799931287765503, | |
| "debug/policy_rejected_logps": -398.26263427734375, | |
| "debug/reference_chosen_logps": -416.07275390625, | |
| "debug/reference_rejected_logps": -398.1192321777344, | |
| "epoch": 0.09375, | |
| "grad_norm": 3.905789416952891, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.734413504600525, | |
| "logits/rejected": 1.5799931287765503, | |
| "logps/chosen": -416.40167236328125, | |
| "logps/rejected": -398.26263427734375, | |
| "loss": 0.5007, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.00328903179615736, | |
| "rewards/margins": -0.0018549349624663591, | |
| "rewards/rejected": -0.0014340970665216446, | |
| "step": 3 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.7133558988571167, | |
| "debug/policy_chosen_logps": -429.8788757324219, | |
| "debug/policy_rejected_logits": 1.561805009841919, | |
| "debug/policy_rejected_logps": -398.11846923828125, | |
| "debug/reference_chosen_logps": -429.8458557128906, | |
| "debug/reference_rejected_logps": -398.2086486816406, | |
| "epoch": 0.125, | |
| "grad_norm": 4.592860856362045, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.7133558988571167, | |
| "logits/rejected": 1.561805009841919, | |
| "logps/chosen": -429.8788757324219, | |
| "logps/rejected": -398.11846923828125, | |
| "loss": 0.5006, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.0003304291167296469, | |
| "rewards/margins": -0.0012322617694735527, | |
| "rewards/rejected": 0.0009018326527439058, | |
| "step": 4 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6188408136367798, | |
| "debug/policy_chosen_logps": -405.05450439453125, | |
| "debug/policy_rejected_logits": 1.7664055824279785, | |
| "debug/policy_rejected_logps": -389.2750244140625, | |
| "debug/reference_chosen_logps": -404.82635498046875, | |
| "debug/reference_rejected_logps": -389.15972900390625, | |
| "epoch": 0.15625, | |
| "grad_norm": 4.105078966015891, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6188408136367798, | |
| "logits/rejected": 1.7664055824279785, | |
| "logps/chosen": -405.05450439453125, | |
| "logps/rejected": -389.2750244140625, | |
| "loss": 0.4992, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.002281513065099716, | |
| "rewards/margins": -0.001129131531342864, | |
| "rewards/rejected": -0.0011523817665874958, | |
| "step": 5 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.5052284002304077, | |
| "debug/policy_chosen_logps": -417.0735778808594, | |
| "debug/policy_rejected_logits": 1.241634488105774, | |
| "debug/policy_rejected_logps": -425.4691467285156, | |
| "debug/reference_chosen_logps": -417.2569580078125, | |
| "debug/reference_rejected_logps": -425.63446044921875, | |
| "epoch": 0.1875, | |
| "grad_norm": 3.9533420030932884, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.5052284002304077, | |
| "logits/rejected": 1.241634488105774, | |
| "logps/chosen": -417.0735778808594, | |
| "logps/rejected": -425.4691467285156, | |
| "loss": 0.5005, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0018336105858907104, | |
| "rewards/margins": 0.0001804353087209165, | |
| "rewards/rejected": 0.0016531753353774548, | |
| "step": 6 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.5048482418060303, | |
| "debug/policy_chosen_logps": -410.84844970703125, | |
| "debug/policy_rejected_logits": 1.9221699237823486, | |
| "debug/policy_rejected_logps": -365.1915283203125, | |
| "debug/reference_chosen_logps": -411.6490478515625, | |
| "debug/reference_rejected_logps": -365.34991455078125, | |
| "epoch": 0.21875, | |
| "grad_norm": 4.1754933609078195, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.5048482418060303, | |
| "logits/rejected": 1.9221699237823486, | |
| "logps/chosen": -410.84844970703125, | |
| "logps/rejected": -365.1915283203125, | |
| "loss": 0.4994, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.008005637675523758, | |
| "rewards/margins": 0.006421546917408705, | |
| "rewards/rejected": 0.0015840912237763405, | |
| "step": 7 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3764795064926147, | |
| "debug/policy_chosen_logps": -410.59136962890625, | |
| "debug/policy_rejected_logits": 1.3361552953720093, | |
| "debug/policy_rejected_logps": -446.6079406738281, | |
| "debug/reference_chosen_logps": -411.0892028808594, | |
| "debug/reference_rejected_logps": -446.5020446777344, | |
| "epoch": 0.25, | |
| "grad_norm": 28.29512163500699, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.3764795064926147, | |
| "logits/rejected": 1.3361552953720093, | |
| "logps/chosen": -410.59136962890625, | |
| "logps/rejected": -446.6079406738281, | |
| "loss": 0.4985, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.004978599026799202, | |
| "rewards/margins": 0.006037559360265732, | |
| "rewards/rejected": -0.0010589603334665298, | |
| "step": 8 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.610563039779663, | |
| "debug/policy_chosen_logps": -406.68865966796875, | |
| "debug/policy_rejected_logits": 1.793952226638794, | |
| "debug/policy_rejected_logps": -407.90484619140625, | |
| "debug/reference_chosen_logps": -406.7731018066406, | |
| "debug/reference_rejected_logps": -408.06695556640625, | |
| "epoch": 0.28125, | |
| "grad_norm": 8.342311302611428, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.610563039779663, | |
| "logits/rejected": 1.793952226638794, | |
| "logps/chosen": -406.68865966796875, | |
| "logps/rejected": -407.90484619140625, | |
| "loss": 0.4981, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": 0.0008446313440799713, | |
| "rewards/margins": -0.0007765005575492978, | |
| "rewards/rejected": 0.0016211319016292691, | |
| "step": 9 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.8192218542099, | |
| "debug/policy_chosen_logps": -399.52435302734375, | |
| "debug/policy_rejected_logits": 1.5653293132781982, | |
| "debug/policy_rejected_logps": -385.7630615234375, | |
| "debug/reference_chosen_logps": -399.35186767578125, | |
| "debug/reference_rejected_logps": -385.9487609863281, | |
| "epoch": 0.3125, | |
| "grad_norm": 4.2502799962912725, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.8192218542099, | |
| "logits/rejected": 1.5653293132781982, | |
| "logps/chosen": -399.52435302734375, | |
| "logps/rejected": -385.7630615234375, | |
| "loss": 0.5017, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0017250444507226348, | |
| "rewards/margins": -0.003581695957109332, | |
| "rewards/rejected": 0.0018566511571407318, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.8145008087158203, | |
| "debug/policy_chosen_logps": -408.01934814453125, | |
| "debug/policy_rejected_logits": 1.7603477239608765, | |
| "debug/policy_rejected_logps": -435.0226135253906, | |
| "debug/reference_chosen_logps": -408.4494323730469, | |
| "debug/reference_rejected_logps": -435.68634033203125, | |
| "epoch": 0.34375, | |
| "grad_norm": 4.1787448496513075, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.8145008087158203, | |
| "logits/rejected": 1.7603477239608765, | |
| "logps/chosen": -408.01934814453125, | |
| "logps/rejected": -435.0226135253906, | |
| "loss": 0.4975, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.004300727508962154, | |
| "rewards/margins": -0.0023362734355032444, | |
| "rewards/rejected": 0.006637001410126686, | |
| "step": 11 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6767597198486328, | |
| "debug/policy_chosen_logps": -384.9023742675781, | |
| "debug/policy_rejected_logits": 1.4636709690093994, | |
| "debug/policy_rejected_logps": -410.29254150390625, | |
| "debug/reference_chosen_logps": -384.634033203125, | |
| "debug/reference_rejected_logps": -410.1116943359375, | |
| "epoch": 0.375, | |
| "grad_norm": 4.319336871459386, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6767597198486328, | |
| "logits/rejected": 1.4636709690093994, | |
| "logps/chosen": -384.9023742675781, | |
| "logps/rejected": -410.29254150390625, | |
| "loss": 0.497, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0026834101881831884, | |
| "rewards/margins": -0.0008746716193854809, | |
| "rewards/rejected": -0.0018087388016283512, | |
| "step": 12 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.508570909500122, | |
| "debug/policy_chosen_logps": -404.47314453125, | |
| "debug/policy_rejected_logits": 1.6251033544540405, | |
| "debug/policy_rejected_logps": -459.6101379394531, | |
| "debug/reference_chosen_logps": -404.60076904296875, | |
| "debug/reference_rejected_logps": -459.2474365234375, | |
| "epoch": 0.40625, | |
| "grad_norm": 4.398229659396608, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.508570909500122, | |
| "logits/rejected": 1.6251033544540405, | |
| "logps/chosen": -404.47314453125, | |
| "logps/rejected": -459.6101379394531, | |
| "loss": 0.4981, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.001276321243494749, | |
| "rewards/margins": 0.004903449676930904, | |
| "rewards/rejected": -0.0036271288990974426, | |
| "step": 13 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.7688783407211304, | |
| "debug/policy_chosen_logps": -424.601318359375, | |
| "debug/policy_rejected_logits": 1.3446720838546753, | |
| "debug/policy_rejected_logps": -452.760009765625, | |
| "debug/reference_chosen_logps": -423.881103515625, | |
| "debug/reference_rejected_logps": -452.08984375, | |
| "epoch": 0.4375, | |
| "grad_norm": 4.8812354492275505, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.7688783407211304, | |
| "logits/rejected": 1.3446720838546753, | |
| "logps/chosen": -424.601318359375, | |
| "logps/rejected": -452.760009765625, | |
| "loss": 0.5007, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.007202301174402237, | |
| "rewards/margins": -0.0005010606837458909, | |
| "rewards/rejected": -0.006701240316033363, | |
| "step": 14 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4991490840911865, | |
| "debug/policy_chosen_logps": -414.95416259765625, | |
| "debug/policy_rejected_logits": 1.7444727420806885, | |
| "debug/policy_rejected_logps": -397.67449951171875, | |
| "debug/reference_chosen_logps": -414.56903076171875, | |
| "debug/reference_rejected_logps": -398.7281494140625, | |
| "epoch": 0.46875, | |
| "grad_norm": 4.614738630368307, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.4991490840911865, | |
| "logits/rejected": 1.7444727420806885, | |
| "logps/chosen": -414.95416259765625, | |
| "logps/rejected": -397.67449951171875, | |
| "loss": 0.5, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.0038512798491865396, | |
| "rewards/margins": -0.014388121664524078, | |
| "rewards/rejected": 0.010536842048168182, | |
| "step": 15 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.8573402166366577, | |
| "debug/policy_chosen_logps": -411.7935485839844, | |
| "debug/policy_rejected_logits": 1.655826449394226, | |
| "debug/policy_rejected_logps": -428.7922668457031, | |
| "debug/reference_chosen_logps": -411.98040771484375, | |
| "debug/reference_rejected_logps": -428.9548645019531, | |
| "epoch": 0.5, | |
| "grad_norm": 4.37108878989768, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.8573402166366577, | |
| "logits/rejected": 1.655826449394226, | |
| "logps/chosen": -411.7935485839844, | |
| "logps/rejected": -428.7922668457031, | |
| "loss": 0.496, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0018686293624341488, | |
| "rewards/margins": 0.0002428237348794937, | |
| "rewards/rejected": 0.0016258049290627241, | |
| "step": 16 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.4710519313812256, | |
| "debug/policy_chosen_logps": -422.3893737792969, | |
| "debug/policy_rejected_logits": 1.6017435789108276, | |
| "debug/policy_rejected_logps": -444.07470703125, | |
| "debug/reference_chosen_logps": -421.87664794921875, | |
| "debug/reference_rejected_logps": -443.59698486328125, | |
| "epoch": 0.53125, | |
| "grad_norm": 3.9347513368146982, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.4710519313812256, | |
| "logits/rejected": 1.6017435789108276, | |
| "logps/chosen": -422.3893737792969, | |
| "logps/rejected": -444.07470703125, | |
| "loss": 0.4916, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.00512706721201539, | |
| "rewards/margins": -0.0003494260599836707, | |
| "rewards/rejected": -0.004777641035616398, | |
| "step": 17 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6341142654418945, | |
| "debug/policy_chosen_logps": -445.0538330078125, | |
| "debug/policy_rejected_logits": 1.6374801397323608, | |
| "debug/policy_rejected_logps": -459.182373046875, | |
| "debug/reference_chosen_logps": -444.14422607421875, | |
| "debug/reference_rejected_logps": -458.5662841796875, | |
| "epoch": 0.5625, | |
| "grad_norm": 4.343729471925386, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6341142654418945, | |
| "logits/rejected": 1.6374801397323608, | |
| "logps/chosen": -445.0538330078125, | |
| "logps/rejected": -459.182373046875, | |
| "loss": 0.4974, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.00909606833010912, | |
| "rewards/margins": -0.0029354477301239967, | |
| "rewards/rejected": -0.006160621531307697, | |
| "step": 18 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.8724883794784546, | |
| "debug/policy_chosen_logps": -373.3547668457031, | |
| "debug/policy_rejected_logits": 1.7754302024841309, | |
| "debug/policy_rejected_logps": -470.7467956542969, | |
| "debug/reference_chosen_logps": -374.51361083984375, | |
| "debug/reference_rejected_logps": -470.7672119140625, | |
| "epoch": 0.59375, | |
| "grad_norm": 4.542983297185189, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.8724883794784546, | |
| "logits/rejected": 1.7754302024841309, | |
| "logps/chosen": -373.3547668457031, | |
| "logps/rejected": -470.7467956542969, | |
| "loss": 0.4987, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.011588573455810547, | |
| "rewards/margins": 0.011384601704776287, | |
| "rewards/rejected": 0.0002039717510342598, | |
| "step": 19 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6738826036453247, | |
| "debug/policy_chosen_logps": -421.78070068359375, | |
| "debug/policy_rejected_logits": 1.6627225875854492, | |
| "debug/policy_rejected_logps": -457.22625732421875, | |
| "debug/reference_chosen_logps": -422.5181884765625, | |
| "debug/reference_rejected_logps": -456.66961669921875, | |
| "epoch": 0.625, | |
| "grad_norm": 4.212961782177093, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6738826036453247, | |
| "logits/rejected": 1.6627225875854492, | |
| "logps/chosen": -421.78070068359375, | |
| "logps/rejected": -457.22625732421875, | |
| "loss": 0.4966, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.007374763488769531, | |
| "rewards/margins": 0.012941397726535797, | |
| "rewards/rejected": -0.0055666351690888405, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6218189001083374, | |
| "debug/policy_chosen_logps": -396.4649658203125, | |
| "debug/policy_rejected_logits": 1.6794122457504272, | |
| "debug/policy_rejected_logps": -393.181640625, | |
| "debug/reference_chosen_logps": -397.00018310546875, | |
| "debug/reference_rejected_logps": -393.26422119140625, | |
| "epoch": 0.65625, | |
| "grad_norm": 4.415514740564342, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6218189001083374, | |
| "logits/rejected": 1.6794122457504272, | |
| "logps/chosen": -396.4649658203125, | |
| "logps/rejected": -393.181640625, | |
| "loss": 0.4973, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0053519438952207565, | |
| "rewards/margins": 0.004526557866483927, | |
| "rewards/rejected": 0.0008253857959061861, | |
| "step": 21 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.8384886980056763, | |
| "debug/policy_chosen_logps": -445.74896240234375, | |
| "debug/policy_rejected_logits": 1.820858359336853, | |
| "debug/policy_rejected_logps": -427.93316650390625, | |
| "debug/reference_chosen_logps": -445.52197265625, | |
| "debug/reference_rejected_logps": -427.1968688964844, | |
| "epoch": 0.6875, | |
| "grad_norm": 4.279507111277066, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.8384886980056763, | |
| "logits/rejected": 1.820858359336853, | |
| "logps/chosen": -445.74896240234375, | |
| "logps/rejected": -427.93316650390625, | |
| "loss": 0.4976, | |
| "rewards/accuracies": 0.375, | |
| "rewards/chosen": -0.002269973512738943, | |
| "rewards/margins": 0.005093079060316086, | |
| "rewards/rejected": -0.007363053038716316, | |
| "step": 22 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.7056151628494263, | |
| "debug/policy_chosen_logps": -412.3712463378906, | |
| "debug/policy_rejected_logits": 1.6649104356765747, | |
| "debug/policy_rejected_logps": -451.2648620605469, | |
| "debug/reference_chosen_logps": -412.66668701171875, | |
| "debug/reference_rejected_logps": -449.33660888671875, | |
| "epoch": 0.71875, | |
| "grad_norm": 5.168198816512977, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.7056151628494263, | |
| "logits/rejected": 1.6649104356765747, | |
| "logps/chosen": -412.3712463378906, | |
| "logps/rejected": -451.2648620605469, | |
| "loss": 0.4922, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.00295448349788785, | |
| "rewards/margins": 0.022237051278352737, | |
| "rewards/rejected": -0.019282570108771324, | |
| "step": 23 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3328857421875, | |
| "debug/policy_chosen_logps": -398.9266357421875, | |
| "debug/policy_rejected_logits": 1.627695083618164, | |
| "debug/policy_rejected_logps": -393.88433837890625, | |
| "debug/reference_chosen_logps": -399.486328125, | |
| "debug/reference_rejected_logps": -395.30322265625, | |
| "epoch": 0.75, | |
| "grad_norm": 5.635074846323111, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.3328857421875, | |
| "logits/rejected": 1.627695083618164, | |
| "logps/chosen": -398.9266357421875, | |
| "logps/rejected": -393.88433837890625, | |
| "loss": 0.5024, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": 0.005597000941634178, | |
| "rewards/margins": -0.00859165284782648, | |
| "rewards/rejected": 0.014188650995492935, | |
| "step": 24 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.1641021966934204, | |
| "debug/policy_chosen_logps": -426.21893310546875, | |
| "debug/policy_rejected_logits": 1.7478457689285278, | |
| "debug/policy_rejected_logps": -509.15692138671875, | |
| "debug/reference_chosen_logps": -426.0503845214844, | |
| "debug/reference_rejected_logps": -508.83636474609375, | |
| "epoch": 0.78125, | |
| "grad_norm": 4.992982311072277, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.1641021966934204, | |
| "logits/rejected": 1.7478457689285278, | |
| "logps/chosen": -426.21893310546875, | |
| "logps/rejected": -509.15692138671875, | |
| "loss": 0.4925, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.00168533343821764, | |
| "rewards/margins": 0.0015203863149508834, | |
| "rewards/rejected": -0.003205719171091914, | |
| "step": 25 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3662775754928589, | |
| "debug/policy_chosen_logps": -370.339111328125, | |
| "debug/policy_rejected_logits": 1.361480951309204, | |
| "debug/policy_rejected_logps": -431.66033935546875, | |
| "debug/reference_chosen_logps": -370.40283203125, | |
| "debug/reference_rejected_logps": -430.5782775878906, | |
| "epoch": 0.8125, | |
| "grad_norm": 5.426783468499563, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.3662775754928589, | |
| "logits/rejected": 1.361480951309204, | |
| "logps/chosen": -370.339111328125, | |
| "logps/rejected": -431.66033935546875, | |
| "loss": 0.4951, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.0006370928604155779, | |
| "rewards/margins": 0.01145759504288435, | |
| "rewards/rejected": -0.01082050334662199, | |
| "step": 26 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.8342163562774658, | |
| "debug/policy_chosen_logps": -379.77557373046875, | |
| "debug/policy_rejected_logits": 1.710644245147705, | |
| "debug/policy_rejected_logps": -458.7286682128906, | |
| "debug/reference_chosen_logps": -381.13714599609375, | |
| "debug/reference_rejected_logps": -457.7099304199219, | |
| "epoch": 0.84375, | |
| "grad_norm": 4.18822192571096, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.8342163562774658, | |
| "logits/rejected": 1.710644245147705, | |
| "logps/chosen": -379.77557373046875, | |
| "logps/rejected": -458.7286682128906, | |
| "loss": 0.4942, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.013616141863167286, | |
| "rewards/margins": 0.02380317449569702, | |
| "rewards/rejected": -0.010187034495174885, | |
| "step": 27 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6289498805999756, | |
| "debug/policy_chosen_logps": -342.238037109375, | |
| "debug/policy_rejected_logits": 1.474778413772583, | |
| "debug/policy_rejected_logps": -447.5248718261719, | |
| "debug/reference_chosen_logps": -343.45904541015625, | |
| "debug/reference_rejected_logps": -446.7528076171875, | |
| "epoch": 0.875, | |
| "grad_norm": 4.120783195357869, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6289498805999756, | |
| "logits/rejected": 1.474778413772583, | |
| "logps/chosen": -342.238037109375, | |
| "logps/rejected": -447.5248718261719, | |
| "loss": 0.4991, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.012210273183882236, | |
| "rewards/margins": 0.019930992275476456, | |
| "rewards/rejected": -0.007720718160271645, | |
| "step": 28 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6164311170578003, | |
| "debug/policy_chosen_logps": -397.0234680175781, | |
| "debug/policy_rejected_logits": 1.5657469034194946, | |
| "debug/policy_rejected_logps": -414.18756103515625, | |
| "debug/reference_chosen_logps": -397.58160400390625, | |
| "debug/reference_rejected_logps": -414.6282958984375, | |
| "epoch": 0.90625, | |
| "grad_norm": 4.352593213110272, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6164311170578003, | |
| "logits/rejected": 1.5657469034194946, | |
| "logps/chosen": -397.0234680175781, | |
| "logps/rejected": -414.18756103515625, | |
| "loss": 0.4981, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.005581206642091274, | |
| "rewards/margins": 0.0011737439781427383, | |
| "rewards/rejected": 0.004407462663948536, | |
| "step": 29 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3324956893920898, | |
| "debug/policy_chosen_logps": -384.20556640625, | |
| "debug/policy_rejected_logits": 1.4190467596054077, | |
| "debug/policy_rejected_logps": -439.9374694824219, | |
| "debug/reference_chosen_logps": -384.3076171875, | |
| "debug/reference_rejected_logps": -439.364990234375, | |
| "epoch": 0.9375, | |
| "grad_norm": 4.24197252932316, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.3324956893920898, | |
| "logits/rejected": 1.4190467596054077, | |
| "logps/chosen": -384.20556640625, | |
| "logps/rejected": -439.9374694824219, | |
| "loss": 0.4902, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.0010205453727394342, | |
| "rewards/margins": 0.006745452992618084, | |
| "rewards/rejected": -0.005724907387048006, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.6599881649017334, | |
| "debug/policy_chosen_logps": -391.8392333984375, | |
| "debug/policy_rejected_logits": 1.440269947052002, | |
| "debug/policy_rejected_logps": -410.36962890625, | |
| "debug/reference_chosen_logps": -391.9766845703125, | |
| "debug/reference_rejected_logps": -410.27911376953125, | |
| "epoch": 0.96875, | |
| "grad_norm": 4.2333369790689614, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.6599881649017334, | |
| "logits/rejected": 1.440269947052002, | |
| "logps/chosen": -391.8392333984375, | |
| "logps/rejected": -410.36962890625, | |
| "loss": 0.4915, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0013742828741669655, | |
| "rewards/margins": 0.002279548905789852, | |
| "rewards/rejected": -0.0009052654495462775, | |
| "step": 31 | |
| }, | |
| { | |
| "debug/policy_chosen_logits": 1.3013134002685547, | |
| "debug/policy_chosen_logps": -468.1767883300781, | |
| "debug/policy_rejected_logits": 1.5036485195159912, | |
| "debug/policy_rejected_logps": -528.7972412109375, | |
| "debug/reference_chosen_logps": -468.2105712890625, | |
| "debug/reference_rejected_logps": -525.7525634765625, | |
| "epoch": 1.0, | |
| "grad_norm": 4.709201002942774, | |
| "learning_rate": 1e-06, | |
| "logits/chosen": 1.3013134002685547, | |
| "logits/rejected": 1.5036485195159912, | |
| "logps/chosen": -468.1767883300781, | |
| "logps/rejected": -528.7972412109375, | |
| "loss": 0.4547, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.00033790571615099907, | |
| "rewards/margins": 0.03078407421708107, | |
| "rewards/rejected": -0.030446168035268784, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 32, | |
| "total_flos": 0.0, | |
| "train_loss": 0.49622934125363827, | |
| "train_runtime": 397.2698, | |
| "train_samples_per_second": 5.022, | |
| "train_steps_per_second": 0.081 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 32, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |