File size: 27,685 Bytes
2d04b70 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 32,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"debug/policy_chosen_logits": 1.5414493083953857,
"debug/policy_chosen_logps": -465.5361328125,
"debug/policy_rejected_logits": 1.6527317762374878,
"debug/policy_rejected_logps": -427.4836120605469,
"debug/reference_chosen_logps": -465.5361328125,
"debug/reference_rejected_logps": -427.4836120605469,
"epoch": 0.03125,
"grad_norm": 12.323765072380743,
"learning_rate": 1e-06,
"logits/chosen": 1.5414493083953857,
"logits/rejected": 1.6527317762374878,
"logps/chosen": -465.5361328125,
"logps/rejected": -427.4836120605469,
"loss": 0.5,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"debug/policy_chosen_logits": 1.8114635944366455,
"debug/policy_chosen_logps": -395.85809326171875,
"debug/policy_rejected_logits": 2.0628278255462646,
"debug/policy_rejected_logps": -405.449462890625,
"debug/reference_chosen_logps": -395.2831726074219,
"debug/reference_rejected_logps": -405.4757080078125,
"epoch": 0.0625,
"grad_norm": 5.688989109980878,
"learning_rate": 1e-06,
"logits/chosen": 1.8114635944366455,
"logits/rejected": 2.0628278255462646,
"logps/chosen": -395.85809326171875,
"logps/rejected": -405.449462890625,
"loss": 0.5021,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.005749321077018976,
"rewards/margins": -0.006011696066707373,
"rewards/rejected": 0.0002623748732730746,
"step": 2
},
{
"debug/policy_chosen_logits": 1.734413504600525,
"debug/policy_chosen_logps": -416.40167236328125,
"debug/policy_rejected_logits": 1.5799931287765503,
"debug/policy_rejected_logps": -398.26263427734375,
"debug/reference_chosen_logps": -416.07275390625,
"debug/reference_rejected_logps": -398.1192321777344,
"epoch": 0.09375,
"grad_norm": 3.905789416952891,
"learning_rate": 1e-06,
"logits/chosen": 1.734413504600525,
"logits/rejected": 1.5799931287765503,
"logps/chosen": -416.40167236328125,
"logps/rejected": -398.26263427734375,
"loss": 0.5007,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.00328903179615736,
"rewards/margins": -0.0018549349624663591,
"rewards/rejected": -0.0014340970665216446,
"step": 3
},
{
"debug/policy_chosen_logits": 1.7133558988571167,
"debug/policy_chosen_logps": -429.8788757324219,
"debug/policy_rejected_logits": 1.561805009841919,
"debug/policy_rejected_logps": -398.11846923828125,
"debug/reference_chosen_logps": -429.8458557128906,
"debug/reference_rejected_logps": -398.2086486816406,
"epoch": 0.125,
"grad_norm": 4.592860856362045,
"learning_rate": 1e-06,
"logits/chosen": 1.7133558988571167,
"logits/rejected": 1.561805009841919,
"logps/chosen": -429.8788757324219,
"logps/rejected": -398.11846923828125,
"loss": 0.5006,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.0003304291167296469,
"rewards/margins": -0.0012322617694735527,
"rewards/rejected": 0.0009018326527439058,
"step": 4
},
{
"debug/policy_chosen_logits": 1.6188408136367798,
"debug/policy_chosen_logps": -405.05450439453125,
"debug/policy_rejected_logits": 1.7664055824279785,
"debug/policy_rejected_logps": -389.2750244140625,
"debug/reference_chosen_logps": -404.82635498046875,
"debug/reference_rejected_logps": -389.15972900390625,
"epoch": 0.15625,
"grad_norm": 4.105078966015891,
"learning_rate": 1e-06,
"logits/chosen": 1.6188408136367798,
"logits/rejected": 1.7664055824279785,
"logps/chosen": -405.05450439453125,
"logps/rejected": -389.2750244140625,
"loss": 0.4992,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.002281513065099716,
"rewards/margins": -0.001129131531342864,
"rewards/rejected": -0.0011523817665874958,
"step": 5
},
{
"debug/policy_chosen_logits": 1.5052284002304077,
"debug/policy_chosen_logps": -417.0735778808594,
"debug/policy_rejected_logits": 1.241634488105774,
"debug/policy_rejected_logps": -425.4691467285156,
"debug/reference_chosen_logps": -417.2569580078125,
"debug/reference_rejected_logps": -425.63446044921875,
"epoch": 0.1875,
"grad_norm": 3.9533420030932884,
"learning_rate": 1e-06,
"logits/chosen": 1.5052284002304077,
"logits/rejected": 1.241634488105774,
"logps/chosen": -417.0735778808594,
"logps/rejected": -425.4691467285156,
"loss": 0.5005,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0018336105858907104,
"rewards/margins": 0.0001804353087209165,
"rewards/rejected": 0.0016531753353774548,
"step": 6
},
{
"debug/policy_chosen_logits": 1.5048482418060303,
"debug/policy_chosen_logps": -410.84844970703125,
"debug/policy_rejected_logits": 1.9221699237823486,
"debug/policy_rejected_logps": -365.1915283203125,
"debug/reference_chosen_logps": -411.6490478515625,
"debug/reference_rejected_logps": -365.34991455078125,
"epoch": 0.21875,
"grad_norm": 4.1754933609078195,
"learning_rate": 1e-06,
"logits/chosen": 1.5048482418060303,
"logits/rejected": 1.9221699237823486,
"logps/chosen": -410.84844970703125,
"logps/rejected": -365.1915283203125,
"loss": 0.4994,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.008005637675523758,
"rewards/margins": 0.006421546917408705,
"rewards/rejected": 0.0015840912237763405,
"step": 7
},
{
"debug/policy_chosen_logits": 1.3764795064926147,
"debug/policy_chosen_logps": -410.59136962890625,
"debug/policy_rejected_logits": 1.3361552953720093,
"debug/policy_rejected_logps": -446.6079406738281,
"debug/reference_chosen_logps": -411.0892028808594,
"debug/reference_rejected_logps": -446.5020446777344,
"epoch": 0.25,
"grad_norm": 28.29512163500699,
"learning_rate": 1e-06,
"logits/chosen": 1.3764795064926147,
"logits/rejected": 1.3361552953720093,
"logps/chosen": -410.59136962890625,
"logps/rejected": -446.6079406738281,
"loss": 0.4985,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.004978599026799202,
"rewards/margins": 0.006037559360265732,
"rewards/rejected": -0.0010589603334665298,
"step": 8
},
{
"debug/policy_chosen_logits": 1.610563039779663,
"debug/policy_chosen_logps": -406.68865966796875,
"debug/policy_rejected_logits": 1.793952226638794,
"debug/policy_rejected_logps": -407.90484619140625,
"debug/reference_chosen_logps": -406.7731018066406,
"debug/reference_rejected_logps": -408.06695556640625,
"epoch": 0.28125,
"grad_norm": 8.342311302611428,
"learning_rate": 1e-06,
"logits/chosen": 1.610563039779663,
"logits/rejected": 1.793952226638794,
"logps/chosen": -406.68865966796875,
"logps/rejected": -407.90484619140625,
"loss": 0.4981,
"rewards/accuracies": 0.375,
"rewards/chosen": 0.0008446313440799713,
"rewards/margins": -0.0007765005575492978,
"rewards/rejected": 0.0016211319016292691,
"step": 9
},
{
"debug/policy_chosen_logits": 1.8192218542099,
"debug/policy_chosen_logps": -399.52435302734375,
"debug/policy_rejected_logits": 1.5653293132781982,
"debug/policy_rejected_logps": -385.7630615234375,
"debug/reference_chosen_logps": -399.35186767578125,
"debug/reference_rejected_logps": -385.9487609863281,
"epoch": 0.3125,
"grad_norm": 4.2502799962912725,
"learning_rate": 1e-06,
"logits/chosen": 1.8192218542099,
"logits/rejected": 1.5653293132781982,
"logps/chosen": -399.52435302734375,
"logps/rejected": -385.7630615234375,
"loss": 0.5017,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.0017250444507226348,
"rewards/margins": -0.003581695957109332,
"rewards/rejected": 0.0018566511571407318,
"step": 10
},
{
"debug/policy_chosen_logits": 1.8145008087158203,
"debug/policy_chosen_logps": -408.01934814453125,
"debug/policy_rejected_logits": 1.7603477239608765,
"debug/policy_rejected_logps": -435.0226135253906,
"debug/reference_chosen_logps": -408.4494323730469,
"debug/reference_rejected_logps": -435.68634033203125,
"epoch": 0.34375,
"grad_norm": 4.1787448496513075,
"learning_rate": 1e-06,
"logits/chosen": 1.8145008087158203,
"logits/rejected": 1.7603477239608765,
"logps/chosen": -408.01934814453125,
"logps/rejected": -435.0226135253906,
"loss": 0.4975,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.004300727508962154,
"rewards/margins": -0.0023362734355032444,
"rewards/rejected": 0.006637001410126686,
"step": 11
},
{
"debug/policy_chosen_logits": 1.6767597198486328,
"debug/policy_chosen_logps": -384.9023742675781,
"debug/policy_rejected_logits": 1.4636709690093994,
"debug/policy_rejected_logps": -410.29254150390625,
"debug/reference_chosen_logps": -384.634033203125,
"debug/reference_rejected_logps": -410.1116943359375,
"epoch": 0.375,
"grad_norm": 4.319336871459386,
"learning_rate": 1e-06,
"logits/chosen": 1.6767597198486328,
"logits/rejected": 1.4636709690093994,
"logps/chosen": -384.9023742675781,
"logps/rejected": -410.29254150390625,
"loss": 0.497,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.0026834101881831884,
"rewards/margins": -0.0008746716193854809,
"rewards/rejected": -0.0018087388016283512,
"step": 12
},
{
"debug/policy_chosen_logits": 1.508570909500122,
"debug/policy_chosen_logps": -404.47314453125,
"debug/policy_rejected_logits": 1.6251033544540405,
"debug/policy_rejected_logps": -459.6101379394531,
"debug/reference_chosen_logps": -404.60076904296875,
"debug/reference_rejected_logps": -459.2474365234375,
"epoch": 0.40625,
"grad_norm": 4.398229659396608,
"learning_rate": 1e-06,
"logits/chosen": 1.508570909500122,
"logits/rejected": 1.6251033544540405,
"logps/chosen": -404.47314453125,
"logps/rejected": -459.6101379394531,
"loss": 0.4981,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.001276321243494749,
"rewards/margins": 0.004903449676930904,
"rewards/rejected": -0.0036271288990974426,
"step": 13
},
{
"debug/policy_chosen_logits": 1.7688783407211304,
"debug/policy_chosen_logps": -424.601318359375,
"debug/policy_rejected_logits": 1.3446720838546753,
"debug/policy_rejected_logps": -452.760009765625,
"debug/reference_chosen_logps": -423.881103515625,
"debug/reference_rejected_logps": -452.08984375,
"epoch": 0.4375,
"grad_norm": 4.8812354492275505,
"learning_rate": 1e-06,
"logits/chosen": 1.7688783407211304,
"logits/rejected": 1.3446720838546753,
"logps/chosen": -424.601318359375,
"logps/rejected": -452.760009765625,
"loss": 0.5007,
"rewards/accuracies": 0.25,
"rewards/chosen": -0.007202301174402237,
"rewards/margins": -0.0005010606837458909,
"rewards/rejected": -0.006701240316033363,
"step": 14
},
{
"debug/policy_chosen_logits": 1.4991490840911865,
"debug/policy_chosen_logps": -414.95416259765625,
"debug/policy_rejected_logits": 1.7444727420806885,
"debug/policy_rejected_logps": -397.67449951171875,
"debug/reference_chosen_logps": -414.56903076171875,
"debug/reference_rejected_logps": -398.7281494140625,
"epoch": 0.46875,
"grad_norm": 4.614738630368307,
"learning_rate": 1e-06,
"logits/chosen": 1.4991490840911865,
"logits/rejected": 1.7444727420806885,
"logps/chosen": -414.95416259765625,
"logps/rejected": -397.67449951171875,
"loss": 0.5,
"rewards/accuracies": 0.25,
"rewards/chosen": -0.0038512798491865396,
"rewards/margins": -0.014388121664524078,
"rewards/rejected": 0.010536842048168182,
"step": 15
},
{
"debug/policy_chosen_logits": 1.8573402166366577,
"debug/policy_chosen_logps": -411.7935485839844,
"debug/policy_rejected_logits": 1.655826449394226,
"debug/policy_rejected_logps": -428.7922668457031,
"debug/reference_chosen_logps": -411.98040771484375,
"debug/reference_rejected_logps": -428.9548645019531,
"epoch": 0.5,
"grad_norm": 4.37108878989768,
"learning_rate": 1e-06,
"logits/chosen": 1.8573402166366577,
"logits/rejected": 1.655826449394226,
"logps/chosen": -411.7935485839844,
"logps/rejected": -428.7922668457031,
"loss": 0.496,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0018686293624341488,
"rewards/margins": 0.0002428237348794937,
"rewards/rejected": 0.0016258049290627241,
"step": 16
},
{
"debug/policy_chosen_logits": 1.4710519313812256,
"debug/policy_chosen_logps": -422.3893737792969,
"debug/policy_rejected_logits": 1.6017435789108276,
"debug/policy_rejected_logps": -444.07470703125,
"debug/reference_chosen_logps": -421.87664794921875,
"debug/reference_rejected_logps": -443.59698486328125,
"epoch": 0.53125,
"grad_norm": 3.9347513368146982,
"learning_rate": 1e-06,
"logits/chosen": 1.4710519313812256,
"logits/rejected": 1.6017435789108276,
"logps/chosen": -422.3893737792969,
"logps/rejected": -444.07470703125,
"loss": 0.4916,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.00512706721201539,
"rewards/margins": -0.0003494260599836707,
"rewards/rejected": -0.004777641035616398,
"step": 17
},
{
"debug/policy_chosen_logits": 1.6341142654418945,
"debug/policy_chosen_logps": -445.0538330078125,
"debug/policy_rejected_logits": 1.6374801397323608,
"debug/policy_rejected_logps": -459.182373046875,
"debug/reference_chosen_logps": -444.14422607421875,
"debug/reference_rejected_logps": -458.5662841796875,
"epoch": 0.5625,
"grad_norm": 4.343729471925386,
"learning_rate": 1e-06,
"logits/chosen": 1.6341142654418945,
"logits/rejected": 1.6374801397323608,
"logps/chosen": -445.0538330078125,
"logps/rejected": -459.182373046875,
"loss": 0.4974,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.00909606833010912,
"rewards/margins": -0.0029354477301239967,
"rewards/rejected": -0.006160621531307697,
"step": 18
},
{
"debug/policy_chosen_logits": 1.8724883794784546,
"debug/policy_chosen_logps": -373.3547668457031,
"debug/policy_rejected_logits": 1.7754302024841309,
"debug/policy_rejected_logps": -470.7467956542969,
"debug/reference_chosen_logps": -374.51361083984375,
"debug/reference_rejected_logps": -470.7672119140625,
"epoch": 0.59375,
"grad_norm": 4.542983297185189,
"learning_rate": 1e-06,
"logits/chosen": 1.8724883794784546,
"logits/rejected": 1.7754302024841309,
"logps/chosen": -373.3547668457031,
"logps/rejected": -470.7467956542969,
"loss": 0.4987,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.011588573455810547,
"rewards/margins": 0.011384601704776287,
"rewards/rejected": 0.0002039717510342598,
"step": 19
},
{
"debug/policy_chosen_logits": 1.6738826036453247,
"debug/policy_chosen_logps": -421.78070068359375,
"debug/policy_rejected_logits": 1.6627225875854492,
"debug/policy_rejected_logps": -457.22625732421875,
"debug/reference_chosen_logps": -422.5181884765625,
"debug/reference_rejected_logps": -456.66961669921875,
"epoch": 0.625,
"grad_norm": 4.212961782177093,
"learning_rate": 1e-06,
"logits/chosen": 1.6738826036453247,
"logits/rejected": 1.6627225875854492,
"logps/chosen": -421.78070068359375,
"logps/rejected": -457.22625732421875,
"loss": 0.4966,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.007374763488769531,
"rewards/margins": 0.012941397726535797,
"rewards/rejected": -0.0055666351690888405,
"step": 20
},
{
"debug/policy_chosen_logits": 1.6218189001083374,
"debug/policy_chosen_logps": -396.4649658203125,
"debug/policy_rejected_logits": 1.6794122457504272,
"debug/policy_rejected_logps": -393.181640625,
"debug/reference_chosen_logps": -397.00018310546875,
"debug/reference_rejected_logps": -393.26422119140625,
"epoch": 0.65625,
"grad_norm": 4.415514740564342,
"learning_rate": 1e-06,
"logits/chosen": 1.6218189001083374,
"logits/rejected": 1.6794122457504272,
"logps/chosen": -396.4649658203125,
"logps/rejected": -393.181640625,
"loss": 0.4973,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0053519438952207565,
"rewards/margins": 0.004526557866483927,
"rewards/rejected": 0.0008253857959061861,
"step": 21
},
{
"debug/policy_chosen_logits": 1.8384886980056763,
"debug/policy_chosen_logps": -445.74896240234375,
"debug/policy_rejected_logits": 1.820858359336853,
"debug/policy_rejected_logps": -427.93316650390625,
"debug/reference_chosen_logps": -445.52197265625,
"debug/reference_rejected_logps": -427.1968688964844,
"epoch": 0.6875,
"grad_norm": 4.279507111277066,
"learning_rate": 1e-06,
"logits/chosen": 1.8384886980056763,
"logits/rejected": 1.820858359336853,
"logps/chosen": -445.74896240234375,
"logps/rejected": -427.93316650390625,
"loss": 0.4976,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.002269973512738943,
"rewards/margins": 0.005093079060316086,
"rewards/rejected": -0.007363053038716316,
"step": 22
},
{
"debug/policy_chosen_logits": 1.7056151628494263,
"debug/policy_chosen_logps": -412.3712463378906,
"debug/policy_rejected_logits": 1.6649104356765747,
"debug/policy_rejected_logps": -451.2648620605469,
"debug/reference_chosen_logps": -412.66668701171875,
"debug/reference_rejected_logps": -449.33660888671875,
"epoch": 0.71875,
"grad_norm": 5.168198816512977,
"learning_rate": 1e-06,
"logits/chosen": 1.7056151628494263,
"logits/rejected": 1.6649104356765747,
"logps/chosen": -412.3712463378906,
"logps/rejected": -451.2648620605469,
"loss": 0.4922,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.00295448349788785,
"rewards/margins": 0.022237051278352737,
"rewards/rejected": -0.019282570108771324,
"step": 23
},
{
"debug/policy_chosen_logits": 1.3328857421875,
"debug/policy_chosen_logps": -398.9266357421875,
"debug/policy_rejected_logits": 1.627695083618164,
"debug/policy_rejected_logps": -393.88433837890625,
"debug/reference_chosen_logps": -399.486328125,
"debug/reference_rejected_logps": -395.30322265625,
"epoch": 0.75,
"grad_norm": 5.635074846323111,
"learning_rate": 1e-06,
"logits/chosen": 1.3328857421875,
"logits/rejected": 1.627695083618164,
"logps/chosen": -398.9266357421875,
"logps/rejected": -393.88433837890625,
"loss": 0.5024,
"rewards/accuracies": 0.25,
"rewards/chosen": 0.005597000941634178,
"rewards/margins": -0.00859165284782648,
"rewards/rejected": 0.014188650995492935,
"step": 24
},
{
"debug/policy_chosen_logits": 1.1641021966934204,
"debug/policy_chosen_logps": -426.21893310546875,
"debug/policy_rejected_logits": 1.7478457689285278,
"debug/policy_rejected_logps": -509.15692138671875,
"debug/reference_chosen_logps": -426.0503845214844,
"debug/reference_rejected_logps": -508.83636474609375,
"epoch": 0.78125,
"grad_norm": 4.992982311072277,
"learning_rate": 1e-06,
"logits/chosen": 1.1641021966934204,
"logits/rejected": 1.7478457689285278,
"logps/chosen": -426.21893310546875,
"logps/rejected": -509.15692138671875,
"loss": 0.4925,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.00168533343821764,
"rewards/margins": 0.0015203863149508834,
"rewards/rejected": -0.003205719171091914,
"step": 25
},
{
"debug/policy_chosen_logits": 1.3662775754928589,
"debug/policy_chosen_logps": -370.339111328125,
"debug/policy_rejected_logits": 1.361480951309204,
"debug/policy_rejected_logps": -431.66033935546875,
"debug/reference_chosen_logps": -370.40283203125,
"debug/reference_rejected_logps": -430.5782775878906,
"epoch": 0.8125,
"grad_norm": 5.426783468499563,
"learning_rate": 1e-06,
"logits/chosen": 1.3662775754928589,
"logits/rejected": 1.361480951309204,
"logps/chosen": -370.339111328125,
"logps/rejected": -431.66033935546875,
"loss": 0.4951,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.0006370928604155779,
"rewards/margins": 0.01145759504288435,
"rewards/rejected": -0.01082050334662199,
"step": 26
},
{
"debug/policy_chosen_logits": 1.8342163562774658,
"debug/policy_chosen_logps": -379.77557373046875,
"debug/policy_rejected_logits": 1.710644245147705,
"debug/policy_rejected_logps": -458.7286682128906,
"debug/reference_chosen_logps": -381.13714599609375,
"debug/reference_rejected_logps": -457.7099304199219,
"epoch": 0.84375,
"grad_norm": 4.18822192571096,
"learning_rate": 1e-06,
"logits/chosen": 1.8342163562774658,
"logits/rejected": 1.710644245147705,
"logps/chosen": -379.77557373046875,
"logps/rejected": -458.7286682128906,
"loss": 0.4942,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.013616141863167286,
"rewards/margins": 0.02380317449569702,
"rewards/rejected": -0.010187034495174885,
"step": 27
},
{
"debug/policy_chosen_logits": 1.6289498805999756,
"debug/policy_chosen_logps": -342.238037109375,
"debug/policy_rejected_logits": 1.474778413772583,
"debug/policy_rejected_logps": -447.5248718261719,
"debug/reference_chosen_logps": -343.45904541015625,
"debug/reference_rejected_logps": -446.7528076171875,
"epoch": 0.875,
"grad_norm": 4.120783195357869,
"learning_rate": 1e-06,
"logits/chosen": 1.6289498805999756,
"logits/rejected": 1.474778413772583,
"logps/chosen": -342.238037109375,
"logps/rejected": -447.5248718261719,
"loss": 0.4991,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.012210273183882236,
"rewards/margins": 0.019930992275476456,
"rewards/rejected": -0.007720718160271645,
"step": 28
},
{
"debug/policy_chosen_logits": 1.6164311170578003,
"debug/policy_chosen_logps": -397.0234680175781,
"debug/policy_rejected_logits": 1.5657469034194946,
"debug/policy_rejected_logps": -414.18756103515625,
"debug/reference_chosen_logps": -397.58160400390625,
"debug/reference_rejected_logps": -414.6282958984375,
"epoch": 0.90625,
"grad_norm": 4.352593213110272,
"learning_rate": 1e-06,
"logits/chosen": 1.6164311170578003,
"logits/rejected": 1.5657469034194946,
"logps/chosen": -397.0234680175781,
"logps/rejected": -414.18756103515625,
"loss": 0.4981,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.005581206642091274,
"rewards/margins": 0.0011737439781427383,
"rewards/rejected": 0.004407462663948536,
"step": 29
},
{
"debug/policy_chosen_logits": 1.3324956893920898,
"debug/policy_chosen_logps": -384.20556640625,
"debug/policy_rejected_logits": 1.4190467596054077,
"debug/policy_rejected_logps": -439.9374694824219,
"debug/reference_chosen_logps": -384.3076171875,
"debug/reference_rejected_logps": -439.364990234375,
"epoch": 0.9375,
"grad_norm": 4.24197252932316,
"learning_rate": 1e-06,
"logits/chosen": 1.3324956893920898,
"logits/rejected": 1.4190467596054077,
"logps/chosen": -384.20556640625,
"logps/rejected": -439.9374694824219,
"loss": 0.4902,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.0010205453727394342,
"rewards/margins": 0.006745452992618084,
"rewards/rejected": -0.005724907387048006,
"step": 30
},
{
"debug/policy_chosen_logits": 1.6599881649017334,
"debug/policy_chosen_logps": -391.8392333984375,
"debug/policy_rejected_logits": 1.440269947052002,
"debug/policy_rejected_logps": -410.36962890625,
"debug/reference_chosen_logps": -391.9766845703125,
"debug/reference_rejected_logps": -410.27911376953125,
"epoch": 0.96875,
"grad_norm": 4.2333369790689614,
"learning_rate": 1e-06,
"logits/chosen": 1.6599881649017334,
"logits/rejected": 1.440269947052002,
"logps/chosen": -391.8392333984375,
"logps/rejected": -410.36962890625,
"loss": 0.4915,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0013742828741669655,
"rewards/margins": 0.002279548905789852,
"rewards/rejected": -0.0009052654495462775,
"step": 31
},
{
"debug/policy_chosen_logits": 1.3013134002685547,
"debug/policy_chosen_logps": -468.1767883300781,
"debug/policy_rejected_logits": 1.5036485195159912,
"debug/policy_rejected_logps": -528.7972412109375,
"debug/reference_chosen_logps": -468.2105712890625,
"debug/reference_rejected_logps": -525.7525634765625,
"epoch": 1.0,
"grad_norm": 4.709201002942774,
"learning_rate": 1e-06,
"logits/chosen": 1.3013134002685547,
"logits/rejected": 1.5036485195159912,
"logps/chosen": -468.1767883300781,
"logps/rejected": -528.7972412109375,
"loss": 0.4547,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.00033790571615099907,
"rewards/margins": 0.03078407421708107,
"rewards/rejected": -0.030446168035268784,
"step": 32
},
{
"epoch": 1.0,
"step": 32,
"total_flos": 0.0,
"train_loss": 0.49622934125363827,
"train_runtime": 397.2698,
"train_samples_per_second": 5.022,
"train_steps_per_second": 0.081
}
],
"logging_steps": 1,
"max_steps": 32,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|