diff --git "a/wandb/run-20250101_112144-t9wzg2aq/files/output.log" "b/wandb/run-20250101_112144-t9wzg2aq/files/output.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20250101_112144-t9wzg2aq/files/output.log" @@ -0,0 +1,1124 @@ +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2025-01-01 11:22:38,142 >> {'loss': 1.4933, 'grad_norm': 33.32280731201172, 'learning_rate': 5.405405405405406e-09, 'epoch': 0.00027122321670735016, 'num_input_tokens_seen': 2097152, 'completed': '0.03% (1 / 3_687)', 'remaining time': '47:29:53', 'throughput': '2825.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:23:13,271 >> {'loss': 1.6295, 'grad_norm': 35.11781692504883, 'learning_rate': 1.0810810810810811e-08, 'epoch': 0.0005424464334147003, 'num_input_tokens_seen': 4194304, 'completed': '0.05% (2 / 3_687)', 'remaining time': '41:43:19', 'throughput': '7462.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:23:48,064 >> {'loss': 1.7417, 'grad_norm': 39.06733703613281, 'learning_rate': 1.6216216216216218e-08, 'epoch': 0.0008136696501220504, 'num_input_tokens_seen': 6291456, 'completed': '0.08% (3 / 3_687)', 'remaining time': '39:40:31', 'throughput': '7534.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:24:19,418 >> {'loss': 1.4688, 'grad_norm': 32.58023452758789, 'learning_rate': 2.1621621621621623e-08, 'epoch': 0.0010848928668294006, 'num_input_tokens_seen': 8388608, 'completed': '0.11% (4 / 3_687)', 'remaining time': '37:46:03', 'throughput': '8360.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:24:51,204 >> {'loss': 2.078, 'grad_norm': 40.15693664550781, 'learning_rate': 2.7027027027027028e-08, 'epoch': 0.0013561160835367507, 'num_input_tokens_seen': 10485760, 'completed': '0.14% (5 / 3_687)', 'remaining time': '36:42:28', 'throughput': '8247.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:25:21,261 >> {'loss': 1.6623, 'grad_norm': 33.44337463378906, 'learning_rate': 3.2432432432432436e-08, 'epoch': 0.0016273393002441008, 'num_input_tokens_seen': 12582912, 'completed': '0.16% (6 / 3_687)', 'remaining time': '35:42:14', 'throughput': '8721.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:25:53,598 >> {'loss': 1.5643, 'grad_norm': 31.053829193115234, 'learning_rate': 3.783783783783784e-08, 'epoch': 0.001898562516951451, 'num_input_tokens_seen': 14680064, 'completed': '0.19% (7 / 3_687)', 'remaining time': '35:19:02', 'throughput': '8106.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:26:24,595 >> {'loss': 1.7671, 'grad_norm': 34.127193450927734, 'learning_rate': 4.3243243243243246e-08, 'epoch': 0.0021697857336588013, 'num_input_tokens_seen': 16777216, 'completed': '0.22% (8 / 3_687)', 'remaining time': '34:51:13', 'throughput': '8457.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:26:54,477 >> {'loss': 2.3356, 'grad_norm': 52.89729309082031, 'learning_rate': 4.864864864864865e-08, 'epoch': 0.0024410089503661514, 'num_input_tokens_seen': 18874368, 'completed': '0.24% (9 / 3_687)', 'remaining time': '34:21:53', 'throughput': '8772.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:27:22,125 >> {'loss': 2.0531, 'grad_norm': 44.45952606201172, 'learning_rate': 5.4054054054054056e-08, 'epoch': 0.0027122321670735015, 'num_input_tokens_seen': 20971520, 'completed': '0.27% (10 / 3_687)', 'remaining time': '33:44:38', 'throughput': '9481.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:27:51,158 >> {'loss': 2.2434, 'grad_norm': 43.71432113647461, 'learning_rate': 5.945945945945946e-08, 'epoch': 0.0029834553837808516, 'num_input_tokens_seen': 23068672, 'completed': '0.30% (11 / 3_687)', 'remaining time': '33:21:47', 'throughput': '9029.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:28:23,581 >> {'loss': 1.5967, 'grad_norm': 35.20856857299805, 'learning_rate': 6.486486486486487e-08, 'epoch': 0.0032546786004882017, 'num_input_tokens_seen': 25165824, 'completed': '0.33% (12 / 3_687)', 'remaining time': '33:19:57', 'throughput': '8085.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:28:53,542 >> {'loss': 1.622, 'grad_norm': 32.52849578857422, 'learning_rate': 7.027027027027027e-08, 'epoch': 0.003525901817195552, 'num_input_tokens_seen': 27262976, 'completed': '0.35% (13 / 3_687)', 'remaining time': '33:06:44', 'throughput': '8749.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:29:26,570 >> {'loss': 2.1261, 'grad_norm': 42.55111312866211, 'learning_rate': 7.567567567567568e-08, 'epoch': 0.003797125033902902, 'num_input_tokens_seen': 29360128, 'completed': '0.38% (14 / 3_687)', 'remaining time': '33:08:44', 'throughput': '7937.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:29:58,803 >> {'loss': 1.102, 'grad_norm': 25.502803802490234, 'learning_rate': 8.108108108108108e-08, 'epoch': 0.0040683482506102524, 'num_input_tokens_seen': 31457280, 'completed': '0.41% (15 / 3_687)', 'remaining time': '33:07:10', 'throughput': '8132.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:30:27,424 >> {'loss': 1.6793, 'grad_norm': 41.81550216674805, 'learning_rate': 8.648648648648649e-08, 'epoch': 0.0043395714673176026, 'num_input_tokens_seen': 33554432, 'completed': '0.43% (16 / 3_687)', 'remaining time': '32:51:54', 'throughput': '9159.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:30:55,989 >> {'loss': 2.5367, 'grad_norm': 51.921199798583984, 'learning_rate': 9.189189189189189e-08, 'epoch': 0.004610794684024953, 'num_input_tokens_seen': 35651584, 'completed': '0.46% (17 / 3_687)', 'remaining time': '32:38:11', 'throughput': '9176.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:31:24,455 >> {'loss': 2.173, 'grad_norm': 39.028221130371094, 'learning_rate': 9.72972972972973e-08, 'epoch': 0.004882017900732303, 'num_input_tokens_seen': 37748736, 'completed': '0.49% (18 / 3_687)', 'remaining time': '32:25:36', 'throughput': '9209.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:31:57,009 >> {'loss': 1.2697, 'grad_norm': 27.318889617919922, 'learning_rate': 1.0270270270270271e-07, 'epoch': 0.005153241117439653, 'num_input_tokens_seen': 39845888, 'completed': '0.52% (19 / 3_687)', 'remaining time': '32:27:26', 'throughput': '8052.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:32:32,038 >> {'loss': 1.1464, 'grad_norm': 24.04189682006836, 'learning_rate': 1.0810810810810811e-07, 'epoch': 0.005424464334147003, 'num_input_tokens_seen': 41943040, 'completed': '0.54% (20 / 3_687)', 'remaining time': '32:36:36', 'throughput': '7483.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:33:02,615 >> {'loss': 1.2522, 'grad_norm': 29.57497787475586, 'learning_rate': 1.1351351351351351e-07, 'epoch': 0.005695687550854353, 'num_input_tokens_seen': 44040192, 'completed': '0.57% (21 / 3_687)', 'remaining time': '32:31:53', 'throughput': '8573.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:33:31,974 >> {'loss': 2.1454, 'grad_norm': 38.306819915771484, 'learning_rate': 1.1891891891891891e-07, 'epoch': 0.005966910767561703, 'num_input_tokens_seen': 46137344, 'completed': '0.60% (22 / 3_687)', 'remaining time': '32:24:10', 'throughput': '8928.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:34:01,498 >> {'loss': 2.2203, 'grad_norm': 39.06193542480469, 'learning_rate': 1.2432432432432432e-07, 'epoch': 0.006238133984269053, 'num_input_tokens_seen': 48234496, 'completed': '0.62% (23 / 3_687)', 'remaining time': '32:17:31', 'throughput': '8879.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:34:29,178 >> {'loss': 2.4594, 'grad_norm': 40.80998229980469, 'learning_rate': 1.2972972972972974e-07, 'epoch': 0.006509357200976403, 'num_input_tokens_seen': 50331648, 'completed': '0.65% (24 / 3_687)', 'remaining time': '32:06:42', 'throughput': '9470.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:35:03,542 >> {'loss': 1.7983, 'grad_norm': 33.7674560546875, 'learning_rate': 1.3513513513513515e-07, 'epoch': 0.0067805804176837535, 'num_input_tokens_seen': 52428800, 'completed': '0.68% (25 / 3_687)', 'remaining time': '32:13:01', 'throughput': '7628.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:35:36,981 >> {'loss': 1.2909, 'grad_norm': 27.341468811035156, 'learning_rate': 1.4054054054054055e-07, 'epoch': 0.007051803634391104, 'num_input_tokens_seen': 54525952, 'completed': '0.71% (26 / 3_687)', 'remaining time': '32:16:38', 'throughput': '7839.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:36:07,640 >> {'loss': 1.0993, 'grad_norm': 26.22152328491211, 'learning_rate': 1.4594594594594595e-07, 'epoch': 0.007323026851098454, 'num_input_tokens_seen': 56623104, 'completed': '0.73% (27 / 3_687)', 'remaining time': '32:13:40', 'throughput': '8550.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:36:41,344 >> {'loss': 1.2144, 'grad_norm': 26.978374481201172, 'learning_rate': 1.5135135135135135e-07, 'epoch': 0.007594250067805804, 'num_input_tokens_seen': 58720256, 'completed': '0.76% (28 / 3_687)', 'remaining time': '32:17:30', 'throughput': '7777.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:37:13,781 >> {'loss': 1.3153, 'grad_norm': 27.742353439331055, 'learning_rate': 1.5675675675675675e-07, 'epoch': 0.007865473284513154, 'num_input_tokens_seen': 60817408, 'completed': '0.79% (29 / 3_687)', 'remaining time': '32:18:22', 'throughput': '8081.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:37:44,423 >> {'loss': 1.9966, 'grad_norm': 36.712318420410156, 'learning_rate': 1.6216216216216215e-07, 'epoch': 0.008136696501220505, 'num_input_tokens_seen': 62914560, 'completed': '0.81% (30 / 3_687)', 'remaining time': '32:15:30', 'throughput': '8554.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:38:15,457 >> {'loss': 1.226, 'grad_norm': 25.96659278869629, 'learning_rate': 1.6756756756756758e-07, 'epoch': 0.008407919717927854, 'num_input_tokens_seen': 65011712, 'completed': '0.84% (31 / 3_687)', 'remaining time': '32:13:33', 'throughput': '8446.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:38:46,959 >> {'loss': 1.3386, 'grad_norm': 27.041015625, 'learning_rate': 1.7297297297297298e-07, 'epoch': 0.008679142934635205, 'num_input_tokens_seen': 67108864, 'completed': '0.87% (32 / 3_687)', 'remaining time': '32:12:35', 'throughput': '8321.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:39:18,739 >> {'loss': 0.9403, 'grad_norm': 21.146181106567383, 'learning_rate': 1.7837837837837838e-07, 'epoch': 0.008950366151342554, 'num_input_tokens_seen': 69206016, 'completed': '0.90% (33 / 3_687)', 'remaining time': '32:12:10', 'throughput': '8248.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:39:53,195 >> {'loss': 1.4175, 'grad_norm': 25.72652244567871, 'learning_rate': 1.8378378378378379e-07, 'epoch': 0.009221589368049905, 'num_input_tokens_seen': 71303168, 'completed': '0.92% (34 / 3_687)', 'remaining time': '32:16:31', 'throughput': '7608.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:40:20,854 >> {'loss': 2.6364, 'grad_norm': 42.69353485107422, 'learning_rate': 1.891891891891892e-07, 'epoch': 0.009492812584757255, 'num_input_tokens_seen': 73400320, 'completed': '0.95% (35 / 3_687)', 'remaining time': '32:08:46', 'throughput': '9477.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:40:53,458 >> {'loss': 1.4978, 'grad_norm': 24.534645080566406, 'learning_rate': 1.945945945945946e-07, 'epoch': 0.009764035801464606, 'num_input_tokens_seen': 75497472, 'completed': '0.98% (36 / 3_687)', 'remaining time': '32:09:48', 'throughput': '8040.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:41:28,437 >> {'loss': 1.2377, 'grad_norm': 22.76692771911621, 'learning_rate': 2e-07, 'epoch': 0.010035259018171955, 'num_input_tokens_seen': 77594624, 'completed': '1.00% (37 / 3_687)', 'remaining time': '32:14:38', 'throughput': '7494.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:41:59,875 >> {'loss': 1.2021, 'grad_norm': 22.586381912231445, 'learning_rate': 2.0540540540540542e-07, 'epoch': 0.010306482234879306, 'num_input_tokens_seen': 79691776, 'completed': '1.03% (38 / 3_687)', 'remaining time': '32:13:31', 'throughput': '8338.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:42:31,650 >> {'loss': 1.0536, 'grad_norm': 24.025348663330078, 'learning_rate': 2.1081081081081082e-07, 'epoch': 0.010577705451586655, 'num_input_tokens_seen': 81788928, 'completed': '1.06% (39 / 3_687)', 'remaining time': '32:12:58', 'throughput': '8250.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:43:02,635 >> {'loss': 1.8307, 'grad_norm': 33.128868103027344, 'learning_rate': 2.1621621621621622e-07, 'epoch': 0.010848928668294006, 'num_input_tokens_seen': 83886080, 'completed': '1.08% (40 / 3_687)', 'remaining time': '32:11:12', 'throughput': '8460.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:43:32,584 >> {'loss': 2.1082, 'grad_norm': 34.85871887207031, 'learning_rate': 2.2162162162162162e-07, 'epoch': 0.011120151885001357, 'num_input_tokens_seen': 85983232, 'completed': '1.11% (41 / 3_687)', 'remaining time': '32:07:58', 'throughput': '8753.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:44:03,995 >> {'loss': 0.9698, 'grad_norm': 22.261695861816406, 'learning_rate': 2.2702702702702703e-07, 'epoch': 0.011391375101708706, 'num_input_tokens_seen': 88080384, 'completed': '1.14% (42 / 3_687)', 'remaining time': '32:06:59', 'throughput': '8345.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:44:33,195 >> {'loss': 1.2482, 'grad_norm': 28.77890396118164, 'learning_rate': 2.3243243243243243e-07, 'epoch': 0.011662598318416057, 'num_input_tokens_seen': 90177536, 'completed': '1.17% (43 / 3_687)', 'remaining time': '32:02:54', 'throughput': '8977.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:45:05,289 >> {'loss': 1.1466, 'grad_norm': 22.587844848632812, 'learning_rate': 2.3783783783783783e-07, 'epoch': 0.011933821535123406, 'num_input_tokens_seen': 92274688, 'completed': '1.19% (44 / 3_687)', 'remaining time': '32:02:58', 'throughput': '8168.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:45:34,968 >> {'loss': 0.9606, 'grad_norm': 18.199480056762695, 'learning_rate': 2.4324324324324326e-07, 'epoch': 0.012205044751830757, 'num_input_tokens_seen': 94371840, 'completed': '1.22% (45 / 3_687)', 'remaining time': '31:59:45', 'throughput': '8832.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:46:03,891 >> {'loss': 1.6163, 'grad_norm': 27.911603927612305, 'learning_rate': 2.4864864864864863e-07, 'epoch': 0.012476267968538107, 'num_input_tokens_seen': 96468992, 'completed': '1.25% (46 / 3_687)', 'remaining time': '31:55:40', 'throughput': '9063.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:46:32,708 >> {'loss': 1.4498, 'grad_norm': 34.57044982910156, 'learning_rate': 2.5405405405405406e-07, 'epoch': 0.012747491185245458, 'num_input_tokens_seen': 98566144, 'completed': '1.27% (47 / 3_687)', 'remaining time': '31:51:35', 'throughput': '9096.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:47:04,683 >> {'loss': 1.3337, 'grad_norm': 21.867979049682617, 'learning_rate': 2.594594594594595e-07, 'epoch': 0.013018714401952807, 'num_input_tokens_seen': 100663296, 'completed': '1.30% (48 / 3_687)', 'remaining time': '31:51:39', 'throughput': '8198.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:47:36,135 >> {'loss': 1.0142, 'grad_norm': 19.342687606811523, 'learning_rate': 2.6486486486486486e-07, 'epoch': 0.013289937618660158, 'num_input_tokens_seen': 102760448, 'completed': '1.33% (49 / 3_687)', 'remaining time': '31:51:02', 'throughput': '8334.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:48:05,707 >> {'loss': 1.0282, 'grad_norm': 24.752653121948242, 'learning_rate': 2.702702702702703e-07, 'epoch': 0.013561160835367507, 'num_input_tokens_seen': 104857600, 'completed': '1.36% (50 / 3_687)', 'remaining time': '31:48:09', 'throughput': '8864.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:48:35,226 >> {'loss': 0.8941, 'grad_norm': 20.70758628845215, 'learning_rate': 2.7567567567567567e-07, 'epoch': 0.013832384052074858, 'num_input_tokens_seen': 106954752, 'completed': '1.38% (51 / 3_687)', 'remaining time': '31:45:18', 'throughput': '8880.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:49:07,602 >> {'loss': 0.8012, 'grad_norm': 22.01601219177246, 'learning_rate': 2.810810810810811e-07, 'epoch': 0.014103607268782207, 'num_input_tokens_seen': 109051904, 'completed': '1.41% (52 / 3_687)', 'remaining time': '31:45:52', 'throughput': '8096.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:49:36,423 >> {'loss': 2.5913, 'grad_norm': 34.209354400634766, 'learning_rate': 2.8648648648648647e-07, 'epoch': 0.014374830485489558, 'num_input_tokens_seen': 111149056, 'completed': '1.44% (53 / 3_687)', 'remaining time': '31:42:19', 'throughput': '9095.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:50:08,534 >> {'loss': 0.9785, 'grad_norm': 19.16649627685547, 'learning_rate': 2.918918918918919e-07, 'epoch': 0.014646053702196907, 'num_input_tokens_seen': 113246208, 'completed': '1.46% (54 / 3_687)', 'remaining time': '31:42:35', 'throughput': '8163.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:50:37,225 >> {'loss': 1.7306, 'grad_norm': 25.85431480407715, 'learning_rate': 2.972972972972973e-07, 'epoch': 0.014917276918904258, 'num_input_tokens_seen': 115343360, 'completed': '1.49% (55 / 3_687)', 'remaining time': '31:39:03', 'throughput': '9136.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:51:06,989 >> {'loss': 0.9716, 'grad_norm': 19.06846809387207, 'learning_rate': 3.027027027027027e-07, 'epoch': 0.015188500135611608, 'num_input_tokens_seen': 117440512, 'completed': '1.52% (56 / 3_687)', 'remaining time': '31:36:48', 'throughput': '8807.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:51:35,008 >> {'loss': 1.4204, 'grad_norm': 23.541532516479492, 'learning_rate': 3.0810810810810813e-07, 'epoch': 0.015459723352318959, 'num_input_tokens_seen': 119537664, 'completed': '1.55% (57 / 3_687)', 'remaining time': '31:32:45', 'throughput': '9355.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:52:06,759 >> {'loss': 0.7799, 'grad_norm': 16.523822784423828, 'learning_rate': 3.135135135135135e-07, 'epoch': 0.015730946569026308, 'num_input_tokens_seen': 121634816, 'completed': '1.57% (58 / 3_687)', 'remaining time': '31:32:43', 'throughput': '8256.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:52:39,696 >> {'loss': 0.9598, 'grad_norm': 21.27098274230957, 'learning_rate': 3.1891891891891893e-07, 'epoch': 0.01600216978573366, 'num_input_tokens_seen': 123731968, 'completed': '1.60% (59 / 3_687)', 'remaining time': '31:33:52', 'throughput': '7959.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:53:09,724 >> {'loss': 0.847, 'grad_norm': 22.250896453857422, 'learning_rate': 3.243243243243243e-07, 'epoch': 0.01627339300244101, 'num_input_tokens_seen': 125829120, 'completed': '1.63% (60 / 3_687)', 'remaining time': '31:32:03', 'throughput': '8730.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:53:38,768 >> {'loss': 1.2704, 'grad_norm': 37.0035514831543, 'learning_rate': 3.2972972972972973e-07, 'epoch': 0.01654461621914836, 'num_input_tokens_seen': 127926272, 'completed': '1.65% (61 / 3_687)', 'remaining time': '31:29:18', 'throughput': '9025.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:54:09,154 >> {'loss': 0.9768, 'grad_norm': 20.06436538696289, 'learning_rate': 3.3513513513513516e-07, 'epoch': 0.016815839435855708, 'num_input_tokens_seen': 130023424, 'completed': '1.68% (62 / 3_687)', 'remaining time': '31:27:55', 'throughput': '8627.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:54:38,681 >> {'loss': 1.1647, 'grad_norm': 20.020830154418945, 'learning_rate': 3.4054054054054054e-07, 'epoch': 0.01708706265256306, 'num_input_tokens_seen': 132120576, 'completed': '1.71% (63 / 3_687)', 'remaining time': '31:25:45', 'throughput': '8878.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:55:08,206 >> {'loss': 0.8014, 'grad_norm': 19.906391143798828, 'learning_rate': 3.4594594594594597e-07, 'epoch': 0.01735828586927041, 'num_input_tokens_seen': 134217728, 'completed': '1.74% (64 / 3_687)', 'remaining time': '31:23:38', 'throughput': '8878.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:55:43,170 >> {'loss': 0.9887, 'grad_norm': 18.05296516418457, 'learning_rate': 3.5135135135135134e-07, 'epoch': 0.01762950908597776, 'num_input_tokens_seen': 136314880, 'completed': '1.76% (65 / 3_687)', 'remaining time': '31:26:36', 'throughput': '7497.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:56:13,171 >> {'loss': 0.9417, 'grad_norm': 18.5097599029541, 'learning_rate': 3.5675675675675677e-07, 'epoch': 0.01790073230268511, 'num_input_tokens_seen': 138412032, 'completed': '1.79% (66 / 3_687)', 'remaining time': '31:24:56', 'throughput': '8737.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:56:43,673 >> {'loss': 1.2979, 'grad_norm': 21.897937774658203, 'learning_rate': 3.6216216216216214e-07, 'epoch': 0.01817195551939246, 'num_input_tokens_seen': 140509184, 'completed': '1.82% (67 / 3_687)', 'remaining time': '31:23:46', 'throughput': '8594.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:57:12,366 >> {'loss': 1.2426, 'grad_norm': 23.45905113220215, 'learning_rate': 3.6756756756756757e-07, 'epoch': 0.01844317873609981, 'num_input_tokens_seen': 142606336, 'completed': '1.84% (68 / 3_687)', 'remaining time': '31:21:00', 'throughput': '9135.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:57:44,334 >> {'loss': 1.4745, 'grad_norm': 21.97277069091797, 'learning_rate': 3.72972972972973e-07, 'epoch': 0.01871440195280716, 'num_input_tokens_seen': 144703488, 'completed': '1.87% (69 / 3_687)', 'remaining time': '31:21:10', 'throughput': '8200.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:58:15,858 >> {'loss': 0.6843, 'grad_norm': 24.767820358276367, 'learning_rate': 3.783783783783784e-07, 'epoch': 0.01898562516951451, 'num_input_tokens_seen': 146800640, 'completed': '1.90% (70 / 3_687)', 'remaining time': '31:20:55', 'throughput': '8315.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:58:44,835 >> {'loss': 1.1348, 'grad_norm': 20.10371971130371, 'learning_rate': 3.837837837837838e-07, 'epoch': 0.01925684838622186, 'num_input_tokens_seen': 148897792, 'completed': '1.93% (71 / 3_687)', 'remaining time': '31:18:31', 'throughput': '9046.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:59:12,917 >> {'loss': 1.7115, 'grad_norm': 24.476518630981445, 'learning_rate': 3.891891891891892e-07, 'epoch': 0.01952807160292921, 'num_input_tokens_seen': 150994944, 'completed': '1.95% (72 / 3_687)', 'remaining time': '31:15:25', 'throughput': '9335.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 11:59:43,328 >> {'loss': 0.7886, 'grad_norm': 19.302143096923828, 'learning_rate': 3.945945945945946e-07, 'epoch': 0.019799294819636562, 'num_input_tokens_seen': 153092096, 'completed': '1.98% (73 / 3_687)', 'remaining time': '31:14:18', 'throughput': '8619.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:00:15,820 >> {'loss': 0.5778, 'grad_norm': 14.634087562561035, 'learning_rate': 4e-07, 'epoch': 0.02007051803634391, 'num_input_tokens_seen': 155189248, 'completed': '2.01% (74 / 3_687)', 'remaining time': '31:14:54', 'throughput': '8067.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:00:45,792 >> {'loss': 0.6952, 'grad_norm': 14.836756706237793, 'learning_rate': 4.054054054054054e-07, 'epoch': 0.02034174125305126, 'num_input_tokens_seen': 157286400, 'completed': '2.03% (75 / 3_687)', 'remaining time': '31:13:27', 'throughput': '8746.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:01:15,855 >> {'loss': 0.7274, 'grad_norm': 15.128747940063477, 'learning_rate': 4.1081081081081084e-07, 'epoch': 0.02061296446975861, 'num_input_tokens_seen': 159383552, 'completed': '2.06% (76 / 3_687)', 'remaining time': '31:12:06', 'throughput': '8719.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:01:46,905 >> {'loss': 0.634, 'grad_norm': 14.638301849365234, 'learning_rate': 4.162162162162162e-07, 'epoch': 0.020884187686465962, 'num_input_tokens_seen': 161480704, 'completed': '2.09% (77 / 3_687)', 'remaining time': '31:11:32', 'throughput': '8442.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:02:15,578 >> {'loss': 1.0147, 'grad_norm': 23.832578659057617, 'learning_rate': 4.2162162162162164e-07, 'epoch': 0.02115541090317331, 'num_input_tokens_seen': 163577856, 'completed': '2.12% (78 / 3_687)', 'remaining time': '31:09:08', 'throughput': '9142.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:02:48,125 >> {'loss': 0.7479, 'grad_norm': 16.2668514251709, 'learning_rate': 4.27027027027027e-07, 'epoch': 0.02142663411988066, 'num_input_tokens_seen': 165675008, 'completed': '2.14% (79 / 3_687)', 'remaining time': '31:09:44', 'throughput': '8054.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:03:21,482 >> {'loss': 0.6993, 'grad_norm': 14.96450424194336, 'learning_rate': 4.3243243243243244e-07, 'epoch': 0.021697857336588012, 'num_input_tokens_seen': 167772160, 'completed': '2.17% (80 / 3_687)', 'remaining time': '31:10:55', 'throughput': '7858.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:03:50,797 >> {'loss': 0.6489, 'grad_norm': 19.316436767578125, 'learning_rate': 4.378378378378378e-07, 'epoch': 0.021969080553295363, 'num_input_tokens_seen': 169869312, 'completed': '2.20% (81 / 3_687)', 'remaining time': '31:09:04', 'throughput': '8942.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:04:20,993 >> {'loss': 0.592, 'grad_norm': 14.418977737426758, 'learning_rate': 4.4324324324324325e-07, 'epoch': 0.022240303770002714, 'num_input_tokens_seen': 171966464, 'completed': '2.22% (82 / 3_687)', 'remaining time': '31:07:53', 'throughput': '8681.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:04:51,863 >> {'loss': 0.7344, 'grad_norm': 19.933923721313477, 'learning_rate': 4.486486486486487e-07, 'epoch': 0.02251152698671006, 'num_input_tokens_seen': 174063616, 'completed': '2.25% (83 / 3_687)', 'remaining time': '31:07:12', 'throughput': '8491.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:05:25,696 >> {'loss': 0.916, 'grad_norm': 20.08726692199707, 'learning_rate': 4.5405405405405405e-07, 'epoch': 0.022782750203417412, 'num_input_tokens_seen': 176160768, 'completed': '2.28% (84 / 3_687)', 'remaining time': '31:08:39', 'throughput': '7748.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:05:56,589 >> {'loss': 0.9287, 'grad_norm': 18.082517623901367, 'learning_rate': 4.594594594594595e-07, 'epoch': 0.023053973420124763, 'num_input_tokens_seen': 178257920, 'completed': '2.31% (85 / 3_687)', 'remaining time': '31:07:58', 'throughput': '8485.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:06:29,559 >> {'loss': 0.8382, 'grad_norm': 15.847949981689453, 'learning_rate': 4.6486486486486485e-07, 'epoch': 0.023325196636832114, 'num_input_tokens_seen': 180355072, 'completed': '2.33% (86 / 3_687)', 'remaining time': '31:08:45', 'throughput': '7951.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:06:59,489 >> {'loss': 1.0967, 'grad_norm': 22.586856842041016, 'learning_rate': 4.702702702702703e-07, 'epoch': 0.023596419853539462, 'num_input_tokens_seen': 182452224, 'completed': '2.36% (87 / 3_687)', 'remaining time': '31:07:24', 'throughput': '8758.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:07:34,459 >> {'loss': 0.675, 'grad_norm': 16.663034439086914, 'learning_rate': 4.7567567567567566e-07, 'epoch': 0.023867643070246813, 'num_input_tokens_seen': 184549376, 'completed': '2.39% (88 / 3_687)', 'remaining time': '31:09:30', 'throughput': '7496.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:08:05,173 >> {'loss': 1.1732, 'grad_norm': 19.878616333007812, 'learning_rate': 4.810810810810811e-07, 'epoch': 0.024138866286954164, 'num_input_tokens_seen': 186646528, 'completed': '2.41% (89 / 3_687)', 'remaining time': '31:08:41', 'throughput': '8534.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:08:37,443 >> {'loss': 0.8503, 'grad_norm': 16.984296798706055, 'learning_rate': 4.864864864864865e-07, 'epoch': 0.024410089503661515, 'num_input_tokens_seen': 188743680, 'completed': '2.44% (90 / 3_687)', 'remaining time': '31:08:54', 'throughput': '8123.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:09:06,835 >> {'loss': 0.4946, 'grad_norm': 14.563621520996094, 'learning_rate': 4.918918918918919e-07, 'epoch': 0.024681312720368862, 'num_input_tokens_seen': 190840832, 'completed': '2.47% (91 / 3_687)', 'remaining time': '31:07:12', 'throughput': '8919.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:09:36,755 >> {'loss': 0.7149, 'grad_norm': 15.319222450256348, 'learning_rate': 4.972972972972973e-07, 'epoch': 0.024952535937076213, 'num_input_tokens_seen': 192937984, 'completed': '2.50% (92 / 3_687)', 'remaining time': '31:05:53', 'throughput': '8761.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:10:06,804 >> {'loss': 0.3925, 'grad_norm': 14.346334457397461, 'learning_rate': 5.027027027027027e-07, 'epoch': 0.025223759153783564, 'num_input_tokens_seen': 195035136, 'completed': '2.52% (93 / 3_687)', 'remaining time': '31:04:39', 'throughput': '8723.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:10:37,080 >> {'loss': 0.9146, 'grad_norm': 19.126708984375, 'learning_rate': 5.081081081081081e-07, 'epoch': 0.025494982370490915, 'num_input_tokens_seen': 197132288, 'completed': '2.55% (94 / 3_687)', 'remaining time': '31:03:35', 'throughput': '8658.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:11:09,397 >> {'loss': 0.8788, 'grad_norm': 18.894250869750977, 'learning_rate': 5.135135135135134e-07, 'epoch': 0.025766205587198263, 'num_input_tokens_seen': 199229440, 'completed': '2.58% (95 / 3_687)', 'remaining time': '31:03:50', 'throughput': '8111.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:11:41,480 >> {'loss': 0.9447, 'grad_norm': 19.886512756347656, 'learning_rate': 5.18918918918919e-07, 'epoch': 0.026037428803905614, 'num_input_tokens_seen': 201326592, 'completed': '2.60% (96 / 3_687)', 'remaining time': '31:03:54', 'throughput': '8170.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:12:10,835 >> {'loss': 0.7017, 'grad_norm': 16.53607940673828, 'learning_rate': 5.243243243243243e-07, 'epoch': 0.026308652020612965, 'num_input_tokens_seen': 203423744, 'completed': '2.63% (97 / 3_687)', 'remaining time': '31:02:17', 'throughput': '8930.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:12:39,542 >> {'loss': 1.0761, 'grad_norm': 21.681087493896484, 'learning_rate': 5.297297297297297e-07, 'epoch': 0.026579875237320316, 'num_input_tokens_seen': 205520896, 'completed': '2.66% (98 / 3_687)', 'remaining time': '31:00:17', 'throughput': '9131.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:13:07,182 >> {'loss': 1.9542, 'grad_norm': 28.16925048828125, 'learning_rate': 5.35135135135135e-07, 'epoch': 0.026851098454027666, 'num_input_tokens_seen': 207618048, 'completed': '2.69% (99 / 3_687)', 'remaining time': '30:57:41', 'throughput': '9484.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:13:37,586 >> {'loss': 0.5452, 'grad_norm': 16.13709831237793, 'learning_rate': 5.405405405405406e-07, 'epoch': 0.027122321670735014, 'num_input_tokens_seen': 209715200, 'completed': '2.71% (100 / 3_687)', 'remaining time': '30:56:46', 'throughput': '8622.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:14:06,924 >> {'loss': 0.4683, 'grad_norm': 12.437922477722168, 'learning_rate': 5.459459459459459e-07, 'epoch': 0.027393544887442365, 'num_input_tokens_seen': 211812352, 'completed': '2.74% (101 / 3_687)', 'remaining time': '30:55:14', 'throughput': '8935.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:14:39,222 >> {'loss': 0.6694, 'grad_norm': 15.048489570617676, 'learning_rate': 5.513513513513513e-07, 'epoch': 0.027664768104149716, 'num_input_tokens_seen': 213909504, 'completed': '2.77% (102 / 3_687)', 'remaining time': '30:55:27', 'throughput': '8116.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:15:08,523 >> {'loss': 0.9275, 'grad_norm': 20.845455169677734, 'learning_rate': 5.567567567567567e-07, 'epoch': 0.027935991320857067, 'num_input_tokens_seen': 216006656, 'completed': '2.79% (103 / 3_687)', 'remaining time': '30:53:55', 'throughput': '8946.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:15:40,888 >> {'loss': 0.6088, 'grad_norm': 15.5338716506958, 'learning_rate': 5.621621621621622e-07, 'epoch': 0.028207214537564414, 'num_input_tokens_seen': 218103808, 'completed': '2.82% (104 / 3_687)', 'remaining time': '30:54:09', 'throughput': '8099.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:16:12,805 >> {'loss': 0.6012, 'grad_norm': 12.675715446472168, 'learning_rate': 5.675675675675675e-07, 'epoch': 0.028478437754271765, 'num_input_tokens_seen': 220200960, 'completed': '2.85% (105 / 3_687)', 'remaining time': '30:54:08', 'throughput': '8213.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:16:42,477 >> {'loss': 0.5751, 'grad_norm': 16.053550720214844, 'learning_rate': 5.729729729729729e-07, 'epoch': 0.028749660970979116, 'num_input_tokens_seen': 222298112, 'completed': '2.87% (106 / 3_687)', 'remaining time': '30:52:50', 'throughput': '8834.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:17:11,892 >> {'loss': 0.8626, 'grad_norm': 18.570253372192383, 'learning_rate': 5.783783783783784e-07, 'epoch': 0.029020884187686467, 'num_input_tokens_seen': 224395264, 'completed': '2.90% (107 / 3_687)', 'remaining time': '30:51:25', 'throughput': '8912.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:17:42,419 >> {'loss': 0.6016, 'grad_norm': 15.664776802062988, 'learning_rate': 5.837837837837838e-07, 'epoch': 0.029292107404393815, 'num_input_tokens_seen': 226492416, 'completed': '2.93% (108 / 3_687)', 'remaining time': '30:50:37', 'throughput': '8587.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:18:13,619 >> {'loss': 0.8534, 'grad_norm': 17.67211151123047, 'learning_rate': 5.891891891891891e-07, 'epoch': 0.029563330621101166, 'num_input_tokens_seen': 228589568, 'completed': '2.96% (109 / 3_687)', 'remaining time': '30:50:12', 'throughput': '8402.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:18:45,709 >> {'loss': 1.1914, 'grad_norm': 23.127309799194336, 'learning_rate': 5.945945945945947e-07, 'epoch': 0.029834553837808517, 'num_input_tokens_seen': 230686720, 'completed': '2.98% (110 / 3_687)', 'remaining time': '30:50:15', 'throughput': '8169.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:19:14,919 >> {'loss': 0.9078, 'grad_norm': 18.636947631835938, 'learning_rate': 6e-07, 'epoch': 0.030105777054515868, 'num_input_tokens_seen': 232783872, 'completed': '3.01% (111 / 3_687)', 'remaining time': '30:48:45', 'throughput': '8974.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:19:43,782 >> {'loss': 1.1244, 'grad_norm': 20.506677627563477, 'learning_rate': 6.054054054054054e-07, 'epoch': 0.030377000271223215, 'num_input_tokens_seen': 234881024, 'completed': '3.04% (112 / 3_687)', 'remaining time': '30:47:05', 'throughput': '9082.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:20:16,389 >> {'loss': 0.5326, 'grad_norm': 15.10786247253418, 'learning_rate': 6.108108108108107e-07, 'epoch': 0.030648223487930566, 'num_input_tokens_seen': 236978176, 'completed': '3.06% (113 / 3_687)', 'remaining time': '30:47:25', 'throughput': '8039.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:20:49,104 >> {'loss': 0.468, 'grad_norm': 14.648728370666504, 'learning_rate': 6.162162162162163e-07, 'epoch': 0.030919446704637917, 'num_input_tokens_seen': 239075328, 'completed': '3.09% (114 / 3_687)', 'remaining time': '30:47:48', 'throughput': '8012.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:21:19,114 >> {'loss': 0.6454, 'grad_norm': 16.831241607666016, 'learning_rate': 6.216216216216216e-07, 'epoch': 0.031190669921345268, 'num_input_tokens_seen': 241172480, 'completed': '3.12% (115 / 3_687)', 'remaining time': '30:46:45', 'throughput': '8735.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:21:48,974 >> {'loss': 0.4922, 'grad_norm': 13.11506175994873, 'learning_rate': 6.27027027027027e-07, 'epoch': 0.031461893138052616, 'num_input_tokens_seen': 243269632, 'completed': '3.15% (116 / 3_687)', 'remaining time': '30:45:38', 'throughput': '8779.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:22:19,547 >> {'loss': 0.6721, 'grad_norm': 18.388965606689453, 'learning_rate': 6.324324324324324e-07, 'epoch': 0.03173311635475997, 'num_input_tokens_seen': 245366784, 'completed': '3.17% (117 / 3_687)', 'remaining time': '30:44:54', 'throughput': '8574.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:22:47,011 >> {'loss': 0.8231, 'grad_norm': 19.308448791503906, 'learning_rate': 6.378378378378379e-07, 'epoch': 0.03200433957146732, 'num_input_tokens_seen': 247463936, 'completed': '3.20% (118 / 3_687)', 'remaining time': '30:42:36', 'throughput': '9544.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:23:15,420 >> {'loss': 1.0588, 'grad_norm': 19.583574295043945, 'learning_rate': 6.432432432432432e-07, 'epoch': 0.032275562788174665, 'num_input_tokens_seen': 249561088, 'completed': '3.23% (119 / 3_687)', 'remaining time': '30:40:48', 'throughput': '9227.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:23:48,996 >> {'loss': 0.5752, 'grad_norm': 15.67451000213623, 'learning_rate': 6.486486486486486e-07, 'epoch': 0.03254678600488202, 'num_input_tokens_seen': 251658240, 'completed': '3.25% (120 / 3_687)', 'remaining time': '30:41:35', 'throughput': '7807.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:24:21,960 >> {'loss': 0.8636, 'grad_norm': 18.46150779724121, 'learning_rate': 6.54054054054054e-07, 'epoch': 0.03281800922158937, 'num_input_tokens_seen': 253755392, 'completed': '3.28% (121 / 3_687)', 'remaining time': '30:42:02', 'throughput': '7952.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:24:52,483 >> {'loss': 0.9201, 'grad_norm': 18.50663948059082, 'learning_rate': 6.594594594594595e-07, 'epoch': 0.03308923243829672, 'num_input_tokens_seen': 255852544, 'completed': '3.31% (122 / 3_687)', 'remaining time': '30:41:17', 'throughput': '8588.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:25:25,791 >> {'loss': 0.7413, 'grad_norm': 18.124380111694336, 'learning_rate': 6.648648648648648e-07, 'epoch': 0.03336045565500407, 'num_input_tokens_seen': 257949696, 'completed': '3.34% (123 / 3_687)', 'remaining time': '30:41:54', 'throughput': '7870.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:25:58,229 >> {'loss': 0.9747, 'grad_norm': 20.973876953125, 'learning_rate': 6.702702702702703e-07, 'epoch': 0.033631678871711417, 'num_input_tokens_seen': 260046848, 'completed': '3.36% (124 / 3_687)', 'remaining time': '30:42:04', 'throughput': '8081.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:26:29,292 >> {'loss': 0.487, 'grad_norm': 15.687307357788086, 'learning_rate': 6.756756756756756e-07, 'epoch': 0.03390290208841877, 'num_input_tokens_seen': 262144000, 'completed': '3.39% (125 / 3_687)', 'remaining time': '30:41:34', 'throughput': '8439.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:27:00,713 >> {'loss': 0.8454, 'grad_norm': 17.15902328491211, 'learning_rate': 6.810810810810811e-07, 'epoch': 0.03417412530512612, 'num_input_tokens_seen': 264241152, 'completed': '3.42% (126 / 3_687)', 'remaining time': '30:41:14', 'throughput': '8342.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:27:30,608 >> {'loss': 0.5951, 'grad_norm': 14.142715454101562, 'learning_rate': 6.864864864864864e-07, 'epoch': 0.034445348521833466, 'num_input_tokens_seen': 266338304, 'completed': '3.44% (127 / 3_687)', 'remaining time': '30:40:12', 'throughput': '8768.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:27:58,503 >> {'loss': 0.9529, 'grad_norm': 20.538616180419922, 'learning_rate': 6.918918918918919e-07, 'epoch': 0.03471657173854082, 'num_input_tokens_seen': 268435456, 'completed': '3.47% (128 / 3_687)', 'remaining time': '30:38:14', 'throughput': '9397.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:28:31,010 >> {'loss': 1.0202, 'grad_norm': 22.79062843322754, 'learning_rate': 6.972972972972973e-07, 'epoch': 0.03498779495524817, 'num_input_tokens_seen': 270532608, 'completed': '3.50% (129 / 3_687)', 'remaining time': '30:38:25', 'throughput': '8064.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:29:03,372 >> {'loss': 0.7667, 'grad_norm': 16.5878963470459, 'learning_rate': 7.027027027027027e-07, 'epoch': 0.03525901817195552, 'num_input_tokens_seen': 272629760, 'completed': '3.53% (130 / 3_687)', 'remaining time': '30:38:31', 'throughput': '8100.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:29:34,795 >> {'loss': 0.6279, 'grad_norm': 16.780086517333984, 'learning_rate': 7.081081081081081e-07, 'epoch': 0.03553024138866287, 'num_input_tokens_seen': 274726912, 'completed': '3.55% (131 / 3_687)', 'remaining time': '30:38:11', 'throughput': '8342.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:30:05,469 >> {'loss': 1.2914, 'grad_norm': 23.6799259185791, 'learning_rate': 7.135135135135135e-07, 'epoch': 0.03580146460537022, 'num_input_tokens_seen': 276824064, 'completed': '3.58% (132 / 3_687)', 'remaining time': '30:37:31', 'throughput': '8546.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:30:36,801 >> {'loss': 0.5839, 'grad_norm': 17.561779022216797, 'learning_rate': 7.189189189189189e-07, 'epoch': 0.03607268782207757, 'num_input_tokens_seen': 278921216, 'completed': '3.61% (133 / 3_687)', 'remaining time': '30:37:08', 'throughput': '8366.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:31:07,758 >> {'loss': 0.7747, 'grad_norm': 21.706342697143555, 'learning_rate': 7.243243243243243e-07, 'epoch': 0.03634391103878492, 'num_input_tokens_seen': 281018368, 'completed': '3.63% (134 / 3_687)', 'remaining time': '30:36:36', 'throughput': '8467.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:31:35,222 >> {'loss': 1.2564, 'grad_norm': 25.93421173095703, 'learning_rate': 7.297297297297297e-07, 'epoch': 0.03661513425549227, 'num_input_tokens_seen': 283115520, 'completed': '3.66% (135 / 3_687)', 'remaining time': '30:34:31', 'throughput': '9545.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:32:05,293 >> {'loss': 0.4117, 'grad_norm': 26.61385154724121, 'learning_rate': 7.351351351351351e-07, 'epoch': 0.03688635747219962, 'num_input_tokens_seen': 285212672, 'completed': '3.69% (136 / 3_687)', 'remaining time': '30:33:36', 'throughput': '8717.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:32:34,290 >> {'loss': 0.6143, 'grad_norm': 19.071443557739258, 'learning_rate': 7.405405405405405e-07, 'epoch': 0.03715758068890697, 'num_input_tokens_seen': 287309824, 'completed': '3.72% (137 / 3_687)', 'remaining time': '30:32:14', 'throughput': '9040.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:33:06,797 >> {'loss': 0.6617, 'grad_norm': 15.275123596191406, 'learning_rate': 7.45945945945946e-07, 'epoch': 0.03742880390561432, 'num_input_tokens_seen': 289406976, 'completed': '3.74% (138 / 3_687)', 'remaining time': '30:32:23', 'throughput': '8064.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:33:37,259 >> {'loss': 0.4291, 'grad_norm': 13.6419095993042, 'learning_rate': 7.513513513513513e-07, 'epoch': 0.03770002712232167, 'num_input_tokens_seen': 291504128, 'completed': '3.77% (139 / 3_687)', 'remaining time': '30:31:38', 'throughput': '8605.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:34:08,259 >> {'loss': 0.5393, 'grad_norm': 14.96066951751709, 'learning_rate': 7.567567567567568e-07, 'epoch': 0.03797125033902902, 'num_input_tokens_seen': 293601280, 'completed': '3.80% (140 / 3_687)', 'remaining time': '30:31:08', 'throughput': '8456.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:34:39,627 >> {'loss': 0.3799, 'grad_norm': 12.42337703704834, 'learning_rate': 7.621621621621621e-07, 'epoch': 0.03824247355573637, 'num_input_tokens_seen': 295698432, 'completed': '3.82% (141 / 3_687)', 'remaining time': '30:30:47', 'throughput': '8356.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:35:11,501 >> {'loss': 0.3632, 'grad_norm': 13.157722473144531, 'learning_rate': 7.675675675675676e-07, 'epoch': 0.03851369677244372, 'num_input_tokens_seen': 297795584, 'completed': '3.85% (142 / 3_687)', 'remaining time': '30:30:38', 'throughput': '8224.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:35:41,631 >> {'loss': 0.7346, 'grad_norm': 16.27894401550293, 'learning_rate': 7.729729729729729e-07, 'epoch': 0.038784919989151075, 'num_input_tokens_seen': 299892736, 'completed': '3.88% (143 / 3_687)', 'remaining time': '30:29:46', 'throughput': '8700.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:36:10,558 >> {'loss': 0.7771, 'grad_norm': 20.3560848236084, 'learning_rate': 7.783783783783784e-07, 'epoch': 0.03905614320585842, 'num_input_tokens_seen': 301989888, 'completed': '3.91% (144 / 3_687)', 'remaining time': '30:28:25', 'throughput': '9062.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:36:41,458 >> {'loss': 1.1799, 'grad_norm': 21.640209197998047, 'learning_rate': 7.837837837837838e-07, 'epoch': 0.03932736642256577, 'num_input_tokens_seen': 304087040, 'completed': '3.93% (145 / 3_687)', 'remaining time': '30:27:52', 'throughput': '8483.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:37:13,815 >> {'loss': 0.5518, 'grad_norm': 14.982572555541992, 'learning_rate': 7.891891891891892e-07, 'epoch': 0.039598589639273124, 'num_input_tokens_seen': 306184192, 'completed': '3.96% (146 / 3_687)', 'remaining time': '30:27:55', 'throughput': '8101.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:37:44,820 >> {'loss': 0.624, 'grad_norm': 18.2089900970459, 'learning_rate': 7.945945945945945e-07, 'epoch': 0.03986981285598047, 'num_input_tokens_seen': 308281344, 'completed': '3.99% (147 / 3_687)', 'remaining time': '30:27:25', 'throughput': '8454.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:38:16,506 >> {'loss': 0.8285, 'grad_norm': 19.922109603881836, 'learning_rate': 8e-07, 'epoch': 0.04014103607268782, 'num_input_tokens_seen': 310378496, 'completed': '4.01% (148 / 3_687)', 'remaining time': '30:27:11', 'throughput': '8273.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:38:46,775 >> {'loss': 0.5316, 'grad_norm': 15.228759765625, 'learning_rate': 8.054054054054054e-07, 'epoch': 0.040412259289395173, 'num_input_tokens_seen': 312475648, 'completed': '4.04% (149 / 3_687)', 'remaining time': '30:26:23', 'throughput': '8660.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:39:20,357 >> {'loss': 0.6752, 'grad_norm': 17.04909324645996, 'learning_rate': 8.108108108108108e-07, 'epoch': 0.04068348250610252, 'num_input_tokens_seen': 314572800, 'completed': '4.07% (150 / 3_687)', 'remaining time': '30:26:54', 'throughput': '7805.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:39:50,785 >> {'loss': 0.8268, 'grad_norm': 18.965408325195312, 'learning_rate': 8.162162162162161e-07, 'epoch': 0.040954705722809875, 'num_input_tokens_seen': 316669952, 'completed': '4.10% (151 / 3_687)', 'remaining time': '30:26:09', 'throughput': '8615.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:40:20,146 >> {'loss': 0.747, 'grad_norm': 18.962045669555664, 'learning_rate': 8.216216216216217e-07, 'epoch': 0.04122592893951722, 'num_input_tokens_seen': 318767104, 'completed': '4.12% (152 / 3_687)', 'remaining time': '30:25:01', 'throughput': '8928.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:40:53,304 >> {'loss': 0.6027, 'grad_norm': 19.22246551513672, 'learning_rate': 8.27027027027027e-07, 'epoch': 0.04149715215622457, 'num_input_tokens_seen': 320864256, 'completed': '4.15% (153 / 3_687)', 'remaining time': '30:25:20', 'throughput': '7906.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:41:25,784 >> {'loss': 0.5226, 'grad_norm': 15.074151039123535, 'learning_rate': 8.324324324324324e-07, 'epoch': 0.041768375372931925, 'num_input_tokens_seen': 322961408, 'completed': '4.18% (154 / 3_687)', 'remaining time': '30:25:23', 'throughput': '8070.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:41:54,166 >> {'loss': 0.9997, 'grad_norm': 21.02098274230957, 'learning_rate': 8.378378378378377e-07, 'epoch': 0.04203959858963927, 'num_input_tokens_seen': 325058560, 'completed': '4.20% (155 / 3_687)', 'remaining time': '30:23:53', 'throughput': '9236.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:42:26,018 >> {'loss': 0.7729, 'grad_norm': 18.864036560058594, 'learning_rate': 8.432432432432433e-07, 'epoch': 0.04231082180634662, 'num_input_tokens_seen': 327155712, 'completed': '4.23% (156 / 3_687)', 'remaining time': '30:23:41', 'throughput': '8229.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:42:55,308 >> {'loss': 0.7608, 'grad_norm': 18.284622192382812, 'learning_rate': 8.486486486486486e-07, 'epoch': 0.042582045023053974, 'num_input_tokens_seen': 329252864, 'completed': '4.26% (157 / 3_687)', 'remaining time': '30:22:32', 'throughput': '8950.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:43:25,707 >> {'loss': 0.7438, 'grad_norm': 16.468151092529297, 'learning_rate': 8.54054054054054e-07, 'epoch': 0.04285326823976132, 'num_input_tokens_seen': 331350016, 'completed': '4.29% (158 / 3_687)', 'remaining time': '30:21:48', 'throughput': '8623.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:43:56,636 >> {'loss': 0.6437, 'grad_norm': 15.429784774780273, 'learning_rate': 8.594594594594595e-07, 'epoch': 0.043124491456468676, 'num_input_tokens_seen': 333447168, 'completed': '4.31% (159 / 3_687)', 'remaining time': '30:21:16', 'throughput': '8475.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:44:27,360 >> {'loss': 1.0021, 'grad_norm': 24.898374557495117, 'learning_rate': 8.648648648648649e-07, 'epoch': 0.043395714673176024, 'num_input_tokens_seen': 335544320, 'completed': '4.34% (160 / 3_687)', 'remaining time': '30:20:40', 'throughput': '8531.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:45:03,189 >> {'loss': 0.5328, 'grad_norm': 15.75402545928955, 'learning_rate': 8.702702702702702e-07, 'epoch': 0.04366693788988337, 'num_input_tokens_seen': 337641472, 'completed': '4.37% (161 / 3_687)', 'remaining time': '30:21:55', 'throughput': '7316.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:45:34,504 >> {'loss': 0.9156, 'grad_norm': 21.882734298706055, 'learning_rate': 8.756756756756756e-07, 'epoch': 0.043938161106590726, 'num_input_tokens_seen': 339738624, 'completed': '4.39% (162 / 3_687)', 'remaining time': '30:21:31', 'throughput': '8371.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:46:02,927 >> {'loss': 1.2029, 'grad_norm': 21.429712295532227, 'learning_rate': 8.810810810810811e-07, 'epoch': 0.04420938432329807, 'num_input_tokens_seen': 341835776, 'completed': '4.42% (163 / 3_687)', 'remaining time': '30:20:04', 'throughput': '9223.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:46:39,129 >> {'loss': 0.3537, 'grad_norm': 12.5687837600708, 'learning_rate': 8.864864864864865e-07, 'epoch': 0.04448060754000543, 'num_input_tokens_seen': 343932928, 'completed': '4.45% (164 / 3_687)', 'remaining time': '30:21:25', 'throughput': '7241.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:47:10,855 >> {'loss': 0.646, 'grad_norm': 19.376272201538086, 'learning_rate': 8.918918918918918e-07, 'epoch': 0.044751830756712775, 'num_input_tokens_seen': 346030080, 'completed': '4.48% (165 / 3_687)', 'remaining time': '30:21:09', 'throughput': '8262.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:47:38,941 >> {'loss': 0.6591, 'grad_norm': 17.1505184173584, 'learning_rate': 8.972972972972974e-07, 'epoch': 0.04502305397342012, 'num_input_tokens_seen': 348127232, 'completed': '4.50% (166 / 3_687)', 'remaining time': '30:19:36', 'throughput': '9333.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:48:09,354 >> {'loss': 0.4214, 'grad_norm': 15.413045883178711, 'learning_rate': 9.027027027027027e-07, 'epoch': 0.04529427719012748, 'num_input_tokens_seen': 350224384, 'completed': '4.53% (167 / 3_687)', 'remaining time': '30:18:52', 'throughput': '8619.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:48:40,320 >> {'loss': 0.6298, 'grad_norm': 17.270416259765625, 'learning_rate': 9.081081081081081e-07, 'epoch': 0.045565500406834825, 'num_input_tokens_seen': 352321536, 'completed': '4.56% (168 / 3_687)', 'remaining time': '30:18:20', 'throughput': '8465.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:49:10,090 >> {'loss': 0.5393, 'grad_norm': 14.51281452178955, 'learning_rate': 9.135135135135134e-07, 'epoch': 0.04583672362354217, 'num_input_tokens_seen': 354418688, 'completed': '4.58% (169 / 3_687)', 'remaining time': '30:17:24', 'throughput': '8805.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:49:41,162 >> {'loss': 0.6491, 'grad_norm': 18.907087326049805, 'learning_rate': 9.18918918918919e-07, 'epoch': 0.04610794684024953, 'num_input_tokens_seen': 356515840, 'completed': '4.61% (170 / 3_687)', 'remaining time': '30:16:54', 'throughput': '8436.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:50:12,070 >> {'loss': 0.5932, 'grad_norm': 17.48542594909668, 'learning_rate': 9.243243243243243e-07, 'epoch': 0.046379170056956874, 'num_input_tokens_seen': 358612992, 'completed': '4.64% (171 / 3_687)', 'remaining time': '30:16:21', 'throughput': '8481.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:50:43,949 >> {'loss': 0.5997, 'grad_norm': 15.82155704498291, 'learning_rate': 9.297297297297297e-07, 'epoch': 0.04665039327366423, 'num_input_tokens_seen': 360710144, 'completed': '4.67% (172 / 3_687)', 'remaining time': '30:16:09', 'throughput': '8223.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:51:17,231 >> {'loss': 0.5147, 'grad_norm': 54.783302307128906, 'learning_rate': 9.351351351351351e-07, 'epoch': 0.046921616490371576, 'num_input_tokens_seen': 362807296, 'completed': '4.69% (173 / 3_687)', 'remaining time': '30:16:24', 'throughput': '7876.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:51:48,892 >> {'loss': 0.5626, 'grad_norm': 15.447076797485352, 'learning_rate': 9.405405405405406e-07, 'epoch': 0.047192839707078924, 'num_input_tokens_seen': 364904448, 'completed': '4.72% (174 / 3_687)', 'remaining time': '30:16:06', 'throughput': '8279.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:52:21,766 >> {'loss': 0.5531, 'grad_norm': 21.660140991210938, 'learning_rate': 9.459459459459459e-07, 'epoch': 0.04746406292378628, 'num_input_tokens_seen': 367001600, 'completed': '4.75% (175 / 3_687)', 'remaining time': '30:16:12', 'throughput': '7974.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:52:49,271 >> {'loss': 0.7348, 'grad_norm': 19.583106994628906, 'learning_rate': 9.513513513513513e-07, 'epoch': 0.047735286140493625, 'num_input_tokens_seen': 369098752, 'completed': '4.77% (176 / 3_687)', 'remaining time': '30:14:31', 'throughput': '9530.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:53:17,434 >> {'loss': 0.7629, 'grad_norm': 19.693716049194336, 'learning_rate': 9.567567567567567e-07, 'epoch': 0.04800650935720097, 'num_input_tokens_seen': 371195904, 'completed': '4.80% (177 / 3_687)', 'remaining time': '30:13:03', 'throughput': '9308.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:53:47,659 >> {'loss': 0.7271, 'grad_norm': 21.72786521911621, 'learning_rate': 9.621621621621622e-07, 'epoch': 0.04827773257390833, 'num_input_tokens_seen': 373293056, 'completed': '4.83% (178 / 3_687)', 'remaining time': '30:12:17', 'throughput': '8672.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:54:19,346 >> {'loss': 0.944, 'grad_norm': 20.64727210998535, 'learning_rate': 9.675675675675676e-07, 'epoch': 0.048548955790615675, 'num_input_tokens_seen': 375390208, 'completed': '4.85% (179 / 3_687)', 'remaining time': '30:12:00', 'throughput': '8273.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:54:52,100 >> {'loss': 0.4082, 'grad_norm': 15.752840042114258, 'learning_rate': 9.72972972972973e-07, 'epoch': 0.04882017900732303, 'num_input_tokens_seen': 377487360, 'completed': '4.88% (180 / 3_687)', 'remaining time': '30:12:03', 'throughput': '8003.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:55:23,477 >> {'loss': 0.5716, 'grad_norm': 16.297456741333008, 'learning_rate': 9.783783783783782e-07, 'epoch': 0.04909140222403038, 'num_input_tokens_seen': 379584512, 'completed': '4.91% (181 / 3_687)', 'remaining time': '30:11:40', 'throughput': '8354.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:55:54,775 >> {'loss': 0.3719, 'grad_norm': 16.411787033081055, 'learning_rate': 9.837837837837839e-07, 'epoch': 0.049362625440737724, 'num_input_tokens_seen': 381681664, 'completed': '4.94% (182 / 3_687)', 'remaining time': '30:11:14', 'throughput': '8375.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:56:27,299 >> {'loss': 0.6982, 'grad_norm': 17.700828552246094, 'learning_rate': 9.89189189189189e-07, 'epoch': 0.04963384865744508, 'num_input_tokens_seen': 383778816, 'completed': '4.96% (183 / 3_687)', 'remaining time': '30:11:12', 'throughput': '8059.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:56:58,879 >> {'loss': 0.4343, 'grad_norm': 13.723708152770996, 'learning_rate': 9.945945945945945e-07, 'epoch': 0.049905071874152426, 'num_input_tokens_seen': 385875968, 'completed': '4.99% (184 / 3_687)', 'remaining time': '30:10:52', 'throughput': '8300.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:57:29,937 >> {'loss': 0.511, 'grad_norm': 18.31014633178711, 'learning_rate': 1e-06, 'epoch': 0.05017629509085978, 'num_input_tokens_seen': 387973120, 'completed': '5.02% (185 / 3_687)', 'remaining time': '30:10:22', 'throughput': '8440.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:58:02,945 >> {'loss': 0.5106, 'grad_norm': 16.061124801635742, 'learning_rate': 9.99999818928562e-07, 'epoch': 0.05044751830756713, 'num_input_tokens_seen': 390070272, 'completed': '5.04% (186 / 3_687)', 'remaining time': '30:10:28', 'throughput': '7941.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:58:32,330 >> {'loss': 0.5723, 'grad_norm': 17.339305877685547, 'learning_rate': 9.999992757143933e-07, 'epoch': 0.050718741524274476, 'num_input_tokens_seen': 392167424, 'completed': '5.07% (187 / 3_687)', 'remaining time': '30:09:26', 'throughput': '8921.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:59:04,457 >> {'loss': 0.5508, 'grad_norm': 15.1113920211792, 'learning_rate': 9.999983703579313e-07, 'epoch': 0.05098996474098183, 'num_input_tokens_seen': 394264576, 'completed': '5.10% (188 / 3_687)', 'remaining time': '30:09:16', 'throughput': '8159.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 12:59:33,616 >> {'loss': 0.7198, 'grad_norm': 18.564332962036133, 'learning_rate': 9.999971028599045e-07, 'epoch': 0.05126118795768918, 'num_input_tokens_seen': 396361728, 'completed': '5.13% (189 / 3_687)', 'remaining time': '30:08:11', 'throughput': '8990.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:00:04,691 >> {'loss': 0.4645, 'grad_norm': 14.539531707763672, 'learning_rate': 9.99995473221333e-07, 'epoch': 0.051532411174396525, 'num_input_tokens_seen': 398458880, 'completed': '5.15% (190 / 3_687)', 'remaining time': '30:07:41', 'throughput': '8435.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:00:36,061 >> {'loss': 0.6121, 'grad_norm': 16.04762077331543, 'learning_rate': 9.999934814435284e-07, 'epoch': 0.05180363439110388, 'num_input_tokens_seen': 400556032, 'completed': '5.18% (191 / 3_687)', 'remaining time': '30:07:16', 'throughput': '8356.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:01:06,133 >> {'loss': 0.6252, 'grad_norm': 17.753110885620117, 'learning_rate': 9.999911275280933e-07, 'epoch': 0.05207485760781123, 'num_input_tokens_seen': 402653184, 'completed': '5.21% (192 / 3_687)', 'remaining time': '30:06:28', 'throughput': '8717.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:01:34,642 >> {'loss': 1.1105, 'grad_norm': 23.137664794921875, 'learning_rate': 9.999884114769223e-07, 'epoch': 0.05234608082451858, 'num_input_tokens_seen': 404750336, 'completed': '5.23% (193 / 3_687)', 'remaining time': '30:05:12', 'throughput': '9195.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:02:05,351 >> {'loss': 0.693, 'grad_norm': 17.16234588623047, 'learning_rate': 9.99985333292201e-07, 'epoch': 0.05261730404122593, 'num_input_tokens_seen': 406847488, 'completed': '5.26% (194 / 3_687)', 'remaining time': '30:04:35', 'throughput': '8536.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:02:35,041 >> {'loss': 1.1388, 'grad_norm': 25.552888870239258, 'learning_rate': 9.999818929764068e-07, 'epoch': 0.05288852725793328, 'num_input_tokens_seen': 408944640, 'completed': '5.29% (195 / 3_687)', 'remaining time': '30:03:41', 'throughput': '8829.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:03:04,652 >> {'loss': 0.5531, 'grad_norm': 17.101301193237305, 'learning_rate': 9.99978090532308e-07, 'epoch': 0.05315975047464063, 'num_input_tokens_seen': 411041792, 'completed': '5.32% (196 / 3_687)', 'remaining time': '30:02:45', 'throughput': '8852.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:03:38,791 >> {'loss': 0.7798, 'grad_norm': 16.60688591003418, 'learning_rate': 9.99973925962965e-07, 'epoch': 0.05343097369134798, 'num_input_tokens_seen': 413138944, 'completed': '5.34% (197 / 3_687)', 'remaining time': '30:03:10', 'throughput': '7678.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:04:09,473 >> {'loss': 0.4118, 'grad_norm': 12.576007843017578, 'learning_rate': 9.999693992717292e-07, 'epoch': 0.05370219690805533, 'num_input_tokens_seen': 415236096, 'completed': '5.37% (198 / 3_687)', 'remaining time': '30:02:34', 'throughput': '8543.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:04:40,967 >> {'loss': 0.4088, 'grad_norm': 12.9966402053833, 'learning_rate': 9.999645104622434e-07, 'epoch': 0.05397342012476268, 'num_input_tokens_seen': 417333248, 'completed': '5.40% (199 / 3_687)', 'remaining time': '30:02:11', 'throughput': '8323.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:05:11,488 >> {'loss': 0.5526, 'grad_norm': 16.39759635925293, 'learning_rate': 9.99959259538442e-07, 'epoch': 0.05424464334147003, 'num_input_tokens_seen': 419430400, 'completed': '5.42% (200 / 3_687)', 'remaining time': '30:01:32', 'throughput': '8589.06', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2025-01-01 13:05:36,124 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-200 +[INFO|configuration_utils.py:472] 2025-01-01 13:05:36,128 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-200/config.json +[INFO|configuration_utils.py:807] 2025-01-01 13:05:36,129 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-200/generation_config.json +[INFO|modeling_utils.py:2807] 2025-01-01 13:06:37,187 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-200/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2025-01-01 13:06:37,190 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-200/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2025-01-01 13:06:37,191 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-200/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2025-01-01 13:10:31,317 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 400, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +01/01/2025 13:10:31 - WARNING - streaming.base.dataset - Because `num_canonical_nodes` was not specified, and `shuffle_algo` is py1e, it will default to be equal to physical nodes. Prior to Streaming v0.7.0, `num_canonical_nodes` defaulted to 64 * physical nodes. +01/01/2025 13:10:31 - WARNING - streaming.base.dataset - Because `shuffle_block_size` was not specified, it will default to max(4_000_000 // num_canonical_nodes, 1 << 18) if num_canonical_nodes is not None, otherwise 262144. Prior to Streaming v0.7.0, `shuffle_block_size` defaulted to 262144. +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2025-01-01 13:11:01,729 >> {'loss': 0.4645, 'grad_norm': 14.523542404174805, 'learning_rate': 9.99953646504551e-07, 'epoch': 0.05451586655817738, 'num_input_tokens_seen': 421527552, 'completed': '5.45% (201 / 3_687)', 'remaining time': '31:33:18', 'throughput': '748.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:11:32,036 >> {'loss': 0.794, 'grad_norm': 17.604217529296875, 'learning_rate': 9.99947671365087e-07, 'epoch': 0.05478708977488473, 'num_input_tokens_seen': 423624704, 'completed': '5.48% (202 / 3_687)', 'remaining time': '31:32:06', 'throughput': '8649.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:12:03,917 >> {'loss': 0.6843, 'grad_norm': 16.45903778076172, 'learning_rate': 9.99941334124859e-07, 'epoch': 0.05505831299159208, 'num_input_tokens_seen': 425721856, 'completed': '5.51% (203 / 3_687)', 'remaining time': '31:31:21', 'throughput': '8222.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:12:35,113 >> {'loss': 0.6436, 'grad_norm': 17.023611068725586, 'learning_rate': 9.999346347889667e-07, 'epoch': 0.05532953620829943, 'num_input_tokens_seen': 427819008, 'completed': '5.53% (204 / 3_687)', 'remaining time': '31:30:25', 'throughput': '8403.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:13:04,115 >> {'loss': 0.707, 'grad_norm': 17.66231346130371, 'learning_rate': 9.999275733628017e-07, 'epoch': 0.05560075942500678, 'num_input_tokens_seen': 429916160, 'completed': '5.56% (205 / 3_687)', 'remaining time': '31:28:52', 'throughput': '9038.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:13:33,272 >> {'loss': 0.5772, 'grad_norm': 14.247184753417969, 'learning_rate': 9.999201498520466e-07, 'epoch': 0.055871982641714134, 'num_input_tokens_seen': 432013312, 'completed': '5.59% (206 / 3_687)', 'remaining time': '31:27:22', 'throughput': '8990.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:14:03,700 >> {'loss': 0.5432, 'grad_norm': 16.290454864501953, 'learning_rate': 9.999123642626758e-07, 'epoch': 0.05614320585842148, 'num_input_tokens_seen': 434110464, 'completed': '5.61% (207 / 3_687)', 'remaining time': '31:26:14', 'throughput': '8615.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:14:33,410 >> {'loss': 0.5976, 'grad_norm': 16.772056579589844, 'learning_rate': 9.999042166009544e-07, 'epoch': 0.05641442907512883, 'num_input_tokens_seen': 436207616, 'completed': '5.64% (208 / 3_687)', 'remaining time': '31:24:55', 'throughput': '8823.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:15:03,839 >> {'loss': 0.3159, 'grad_norm': 10.778910636901855, 'learning_rate': 9.998957068734399e-07, 'epoch': 0.05668565229183618, 'num_input_tokens_seen': 438304768, 'completed': '5.67% (209 / 3_687)', 'remaining time': '31:23:48', 'throughput': '8614.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:15:33,470 >> {'loss': 0.9222, 'grad_norm': 21.58819580078125, 'learning_rate': 9.9988683508698e-07, 'epoch': 0.05695687550854353, 'num_input_tokens_seen': 440401920, 'completed': '5.70% (210 / 3_687)', 'remaining time': '31:22:28', 'throughput': '8846.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:16:04,866 >> {'loss': 1.0365, 'grad_norm': 21.694862365722656, 'learning_rate': 9.99877601248715e-07, 'epoch': 0.05722809872525088, 'num_input_tokens_seen': 442499072, 'completed': '5.72% (211 / 3_687)', 'remaining time': '31:21:37', 'throughput': '8349.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:16:37,108 >> {'loss': 0.5279, 'grad_norm': 20.887956619262695, 'learning_rate': 9.998680053660756e-07, 'epoch': 0.05749932194195823, 'num_input_tokens_seen': 444596224, 'completed': '5.75% (212 / 3_687)', 'remaining time': '31:21:01', 'throughput': '8130.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:17:09,120 >> {'loss': 0.7565, 'grad_norm': 20.843788146972656, 'learning_rate': 9.998580474467842e-07, 'epoch': 0.05777054515866558, 'num_input_tokens_seen': 446693376, 'completed': '5.78% (213 / 3_687)', 'remaining time': '31:20:21', 'throughput': '8188.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:17:38,655 >> {'loss': 0.4137, 'grad_norm': 13.617545127868652, 'learning_rate': 9.998477274988545e-07, 'epoch': 0.058041768375372935, 'num_input_tokens_seen': 448790528, 'completed': '5.80% (214 / 3_687)', 'remaining time': '31:19:01', 'throughput': '8875.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:18:11,142 >> {'loss': 0.4458, 'grad_norm': 11.583077430725098, 'learning_rate': 9.998370455305918e-07, 'epoch': 0.05831299159208028, 'num_input_tokens_seen': 450887680, 'completed': '5.83% (215 / 3_687)', 'remaining time': '31:18:29', 'throughput': '8069.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:18:41,706 >> {'loss': 0.6287, 'grad_norm': 16.68201446533203, 'learning_rate': 9.998260015505923e-07, 'epoch': 0.05858421480878763, 'num_input_tokens_seen': 452984832, 'completed': '5.86% (216 / 3_687)', 'remaining time': '31:17:26', 'throughput': '8576.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:19:11,894 >> {'loss': 0.4641, 'grad_norm': 13.695748329162598, 'learning_rate': 9.998145955677438e-07, 'epoch': 0.058855438025494984, 'num_input_tokens_seen': 455081984, 'completed': '5.89% (217 / 3_687)', 'remaining time': '31:16:17', 'throughput': '8683.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:19:46,634 >> {'loss': 0.3928, 'grad_norm': 12.062211990356445, 'learning_rate': 9.998028275912257e-07, 'epoch': 0.05912666124220233, 'num_input_tokens_seen': 457179136, 'completed': '5.91% (218 / 3_687)', 'remaining time': '31:16:21', 'throughput': '7545.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:20:17,844 >> {'loss': 0.5814, 'grad_norm': 19.77566146850586, 'learning_rate': 9.997906976305082e-07, 'epoch': 0.059397884458909686, 'num_input_tokens_seen': 459276288, 'completed': '5.94% (219 / 3_687)', 'remaining time': '31:15:29', 'throughput': '8399.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:20:47,575 >> {'loss': 0.8212, 'grad_norm': 19.376760482788086, 'learning_rate': 9.99778205695353e-07, 'epoch': 0.059669107675617034, 'num_input_tokens_seen': 461373440, 'completed': '5.97% (220 / 3_687)', 'remaining time': '31:14:14', 'throughput': '8816.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:21:18,206 >> {'loss': 0.4862, 'grad_norm': 13.375431060791016, 'learning_rate': 9.997653517958132e-07, 'epoch': 0.05994033089232438, 'num_input_tokens_seen': 463470592, 'completed': '5.99% (221 / 3_687)', 'remaining time': '31:13:13', 'throughput': '8558.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:21:50,825 >> {'loss': 0.6401, 'grad_norm': 16.914608001708984, 'learning_rate': 9.997521359422332e-07, 'epoch': 0.060211554109031735, 'num_input_tokens_seen': 465567744, 'completed': '6.02% (222 / 3_687)', 'remaining time': '31:12:43', 'throughput': '8036.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:22:20,936 >> {'loss': 0.4319, 'grad_norm': 13.095565795898438, 'learning_rate': 9.997385581452484e-07, 'epoch': 0.06048277732573908, 'num_input_tokens_seen': 467664896, 'completed': '6.05% (223 / 3_687)', 'remaining time': '31:11:35', 'throughput': '8705.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:22:51,726 >> {'loss': 0.3974, 'grad_norm': 15.450024604797363, 'learning_rate': 9.99724618415786e-07, 'epoch': 0.06075400054244643, 'num_input_tokens_seen': 469762048, 'completed': '6.08% (224 / 3_687)', 'remaining time': '31:10:37', 'throughput': '8514.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:23:20,594 >> {'loss': 0.7345, 'grad_norm': 15.416088104248047, 'learning_rate': 9.997103167650637e-07, 'epoch': 0.061025223759153785, 'num_input_tokens_seen': 471859200, 'completed': '6.10% (225 / 3_687)', 'remaining time': '31:09:10', 'throughput': '9080.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:23:53,276 >> {'loss': 0.4602, 'grad_norm': 12.959712982177734, 'learning_rate': 9.996956532045914e-07, 'epoch': 0.06129644697586113, 'num_input_tokens_seen': 473956352, 'completed': '6.13% (226 / 3_687)', 'remaining time': '31:08:42', 'throughput': '8020.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:24:22,853 >> {'loss': 0.9696, 'grad_norm': 24.262161254882812, 'learning_rate': 9.996806277461696e-07, 'epoch': 0.06156767019256849, 'num_input_tokens_seen': 476053504, 'completed': '6.16% (227 / 3_687)', 'remaining time': '31:07:27', 'throughput': '8863.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:24:52,100 >> {'loss': 0.7863, 'grad_norm': 21.882034301757812, 'learning_rate': 9.9966524040189e-07, 'epoch': 0.061838893409275834, 'num_input_tokens_seen': 478150656, 'completed': '6.18% (228 / 3_687)', 'remaining time': '31:06:07', 'throughput': '8962.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:25:21,331 >> {'loss': 0.6471, 'grad_norm': 17.270530700683594, 'learning_rate': 9.996494911841363e-07, 'epoch': 0.06211011662598318, 'num_input_tokens_seen': 480247808, 'completed': '6.21% (229 / 3_687)', 'remaining time': '31:04:47', 'throughput': '8968.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:25:51,861 >> {'loss': 0.3576, 'grad_norm': 13.247891426086426, 'learning_rate': 9.996333801055823e-07, 'epoch': 0.062381339842690536, 'num_input_tokens_seen': 482344960, 'completed': '6.24% (230 / 3_687)', 'remaining time': '31:03:48', 'throughput': '8586.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:26:24,018 >> {'loss': 0.4596, 'grad_norm': 13.374452590942383, 'learning_rate': 9.99616907179194e-07, 'epoch': 0.06265256305939788, 'num_input_tokens_seen': 484442112, 'completed': '6.27% (231 / 3_687)', 'remaining time': '31:03:12', 'throughput': '8151.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:26:54,109 >> {'loss': 0.7839, 'grad_norm': 18.631214141845703, 'learning_rate': 9.996000724182278e-07, 'epoch': 0.06292378627610523, 'num_input_tokens_seen': 486539264, 'completed': '6.29% (232 / 3_687)', 'remaining time': '31:02:06', 'throughput': '8711.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:27:22,302 >> {'loss': 0.636, 'grad_norm': 18.132368087768555, 'learning_rate': 9.99582875836232e-07, 'epoch': 0.06319500949281258, 'num_input_tokens_seen': 488636416, 'completed': '6.32% (233 / 3_687)', 'remaining time': '31:00:33', 'throughput': '9298.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:27:52,198 >> {'loss': 0.6983, 'grad_norm': 15.695167541503906, 'learning_rate': 9.995653174470456e-07, 'epoch': 0.06346623270951994, 'num_input_tokens_seen': 490733568, 'completed': '6.35% (234 / 3_687)', 'remaining time': '30:59:25', 'throughput': '8768.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:28:23,405 >> {'loss': 0.4567, 'grad_norm': 13.286776542663574, 'learning_rate': 9.99547397264799e-07, 'epoch': 0.06373745592622729, 'num_input_tokens_seen': 492830720, 'completed': '6.37% (235 / 3_687)', 'remaining time': '30:58:36', 'throughput': '8400.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:28:52,459 >> {'loss': 0.359, 'grad_norm': 11.679216384887695, 'learning_rate': 9.995291153039135e-07, 'epoch': 0.06400867914293464, 'num_input_tokens_seen': 494927872, 'completed': '6.40% (236 / 3_687)', 'remaining time': '30:57:16', 'throughput': '9022.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:29:22,044 >> {'loss': 0.6919, 'grad_norm': 16.616655349731445, 'learning_rate': 9.99510471579102e-07, 'epoch': 0.06427990235964198, 'num_input_tokens_seen': 497025024, 'completed': '6.43% (237 / 3_687)', 'remaining time': '30:56:05', 'throughput': '8860.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:29:55,413 >> {'loss': 0.5059, 'grad_norm': 15.427544593811035, 'learning_rate': 9.99491466105368e-07, 'epoch': 0.06455112557634933, 'num_input_tokens_seen': 499122176, 'completed': '6.46% (238 / 3_687)', 'remaining time': '30:55:48', 'throughput': '7855.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:30:30,184 >> {'loss': 0.6509, 'grad_norm': 38.47232437133789, 'learning_rate': 9.994720988980065e-07, 'epoch': 0.06482234879305669, 'num_input_tokens_seen': 501219328, 'completed': '6.48% (239 / 3_687)', 'remaining time': '30:55:52', 'throughput': '7539.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:31:02,405 >> {'loss': 0.3664, 'grad_norm': 12.464300155639648, 'learning_rate': 9.994523699726035e-07, 'epoch': 0.06509357200976404, 'num_input_tokens_seen': 503316480, 'completed': '6.51% (240 / 3_687)', 'remaining time': '30:55:18', 'throughput': '8135.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:31:30,615 >> {'loss': 0.9145, 'grad_norm': 25.51685905456543, 'learning_rate': 9.994322793450361e-07, 'epoch': 0.06536479522647139, 'num_input_tokens_seen': 505413632, 'completed': '6.54% (241 / 3_687)', 'remaining time': '30:53:48', 'throughput': '9292.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:32:00,211 >> {'loss': 0.7751, 'grad_norm': 18.906505584716797, 'learning_rate': 9.994118270314725e-07, 'epoch': 0.06563601844317873, 'num_input_tokens_seen': 507510784, 'completed': '6.56% (242 / 3_687)', 'remaining time': '30:52:37', 'throughput': '8857.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:32:28,848 >> {'loss': 0.8191, 'grad_norm': 22.564781188964844, 'learning_rate': 9.993910130483717e-07, 'epoch': 0.06590724165988608, 'num_input_tokens_seen': 509607936, 'completed': '6.59% (243 / 3_687)', 'remaining time': '30:51:13', 'throughput': '9154.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:32:59,622 >> {'loss': 0.4122, 'grad_norm': 12.493294715881348, 'learning_rate': 9.993698374124844e-07, 'epoch': 0.06617846487659344, 'num_input_tokens_seen': 511705088, 'completed': '6.62% (244 / 3_687)', 'remaining time': '30:50:20', 'throughput': '8518.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:33:28,257 >> {'loss': 0.6446, 'grad_norm': 16.386438369750977, 'learning_rate': 9.993483001408516e-07, 'epoch': 0.06644968809330079, 'num_input_tokens_seen': 513802240, 'completed': '6.64% (245 / 3_687)', 'remaining time': '30:48:57', 'throughput': '9154.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:33:57,706 >> {'loss': 0.7367, 'grad_norm': 20.80731773376465, 'learning_rate': 9.99326401250806e-07, 'epoch': 0.06672091131000814, 'num_input_tokens_seen': 515899392, 'completed': '6.67% (246 / 3_687)', 'remaining time': '30:47:46', 'throughput': '8901.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:34:27,700 >> {'loss': 0.5971, 'grad_norm': 21.987316131591797, 'learning_rate': 9.993041407599708e-07, 'epoch': 0.06699213452671549, 'num_input_tokens_seen': 517996544, 'completed': '6.70% (247 / 3_687)', 'remaining time': '30:46:43', 'throughput': '8739.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:34:57,333 >> {'loss': 0.468, 'grad_norm': 14.191584587097168, 'learning_rate': 9.992815186862602e-07, 'epoch': 0.06726335774342283, 'num_input_tokens_seen': 520093696, 'completed': '6.73% (248 / 3_687)', 'remaining time': '30:45:35', 'throughput': '8846.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:35:32,375 >> {'loss': 0.7622, 'grad_norm': 19.399492263793945, 'learning_rate': 9.9925853504788e-07, 'epoch': 0.06753458096013018, 'num_input_tokens_seen': 522190848, 'completed': '6.75% (249 / 3_687)', 'remaining time': '30:45:42', 'throughput': '7480.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:36:04,486 >> {'loss': 0.6956, 'grad_norm': 15.56145191192627, 'learning_rate': 9.992351898633262e-07, 'epoch': 0.06780580417683754, 'num_input_tokens_seen': 524288000, 'completed': '6.78% (250 / 3_687)', 'remaining time': '30:45:08', 'throughput': '8163.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:36:36,985 >> {'loss': 0.2901, 'grad_norm': 10.482128143310547, 'learning_rate': 9.992114831513863e-07, 'epoch': 0.06807702739354489, 'num_input_tokens_seen': 526385152, 'completed': '6.81% (251 / 3_687)', 'remaining time': '30:44:40', 'throughput': '8066.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:37:06,097 >> {'loss': 0.5265, 'grad_norm': 14.247305870056152, 'learning_rate': 9.991874149311386e-07, 'epoch': 0.06834825061025224, 'num_input_tokens_seen': 528482304, 'completed': '6.83% (252 / 3_687)', 'remaining time': '30:43:26', 'throughput': '9004.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:37:34,636 >> {'loss': 1.0187, 'grad_norm': 23.847684860229492, 'learning_rate': 9.991629852219523e-07, 'epoch': 0.06861947382695958, 'num_input_tokens_seen': 530579456, 'completed': '6.86% (253 / 3_687)', 'remaining time': '30:42:04', 'throughput': '9185.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:38:05,153 >> {'loss': 0.7402, 'grad_norm': 19.322067260742188, 'learning_rate': 9.991381940434873e-07, 'epoch': 0.06889069704366693, 'num_input_tokens_seen': 532676608, 'completed': '6.89% (254 / 3_687)', 'remaining time': '30:41:09', 'throughput': '8590.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:38:33,588 >> {'loss': 0.8914, 'grad_norm': 19.398937225341797, 'learning_rate': 9.991130414156946e-07, 'epoch': 0.0691619202603743, 'num_input_tokens_seen': 534773760, 'completed': '6.92% (255 / 3_687)', 'remaining time': '30:39:47', 'throughput': '9219.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:39:04,770 >> {'loss': 0.6505, 'grad_norm': 14.539494514465332, 'learning_rate': 9.990875273588161e-07, 'epoch': 0.06943314347708164, 'num_input_tokens_seen': 536870912, 'completed': '6.94% (256 / 3_687)', 'remaining time': '30:39:01', 'throughput': '8407.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:39:36,564 >> {'loss': 0.6311, 'grad_norm': 15.979605674743652, 'learning_rate': 9.99061651893385e-07, 'epoch': 0.06970436669378899, 'num_input_tokens_seen': 538968064, 'completed': '6.97% (257 / 3_687)', 'remaining time': '30:38:24', 'throughput': '8245.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:40:08,660 >> {'loss': 0.4779, 'grad_norm': 13.959792137145996, 'learning_rate': 9.990354150402242e-07, 'epoch': 0.06997558991049634, 'num_input_tokens_seen': 541065216, 'completed': '7.00% (258 / 3_687)', 'remaining time': '30:37:51', 'throughput': '8167.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:40:38,618 >> {'loss': 0.4992, 'grad_norm': 14.552385330200195, 'learning_rate': 9.990088168204487e-07, 'epoch': 0.07024681312720368, 'num_input_tokens_seen': 543162368, 'completed': '7.02% (259 / 3_687)', 'remaining time': '30:36:50', 'throughput': '8750.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:41:07,821 >> {'loss': 0.8364, 'grad_norm': 18.06981086730957, 'learning_rate': 9.989818572554633e-07, 'epoch': 0.07051803634391104, 'num_input_tokens_seen': 545259520, 'completed': '7.05% (260 / 3_687)', 'remaining time': '30:35:39', 'throughput': '8976.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:41:38,948 >> {'loss': 0.7462, 'grad_norm': 16.378019332885742, 'learning_rate': 9.989545363669644e-07, 'epoch': 0.07078925956061839, 'num_input_tokens_seen': 547356672, 'completed': '7.08% (261 / 3_687)', 'remaining time': '30:34:53', 'throughput': '8421.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:42:10,631 >> {'loss': 0.4064, 'grad_norm': 12.575121879577637, 'learning_rate': 9.989268541769383e-07, 'epoch': 0.07106048277732574, 'num_input_tokens_seen': 549453824, 'completed': '7.11% (262 / 3_687)', 'remaining time': '30:34:15', 'throughput': '8274.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:42:41,369 >> {'loss': 0.6201, 'grad_norm': 14.232921600341797, 'learning_rate': 9.988988107076632e-07, 'epoch': 0.07133170599403309, 'num_input_tokens_seen': 551550976, 'completed': '7.13% (263 / 3_687)', 'remaining time': '30:33:25', 'throughput': '8528.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:43:13,197 >> {'loss': 0.4924, 'grad_norm': 14.675073623657227, 'learning_rate': 9.98870405981707e-07, 'epoch': 0.07160292921074043, 'num_input_tokens_seen': 553648128, 'completed': '7.16% (264 / 3_687)', 'remaining time': '30:32:49', 'throughput': '8236.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:43:44,397 >> {'loss': 0.6525, 'grad_norm': 15.136611938476562, 'learning_rate': 9.988416400219288e-07, 'epoch': 0.0718741524274478, 'num_input_tokens_seen': 555745280, 'completed': '7.19% (265 / 3_687)', 'remaining time': '30:32:05', 'throughput': '8402.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:44:15,790 >> {'loss': 0.8809, 'grad_norm': 18.230609893798828, 'learning_rate': 9.988125128514785e-07, 'epoch': 0.07214537564415514, 'num_input_tokens_seen': 557842432, 'completed': '7.21% (266 / 3_687)', 'remaining time': '30:31:24', 'throughput': '8350.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:44:45,507 >> {'loss': 0.8042, 'grad_norm': 15.418828964233398, 'learning_rate': 9.987830244937964e-07, 'epoch': 0.07241659886086249, 'num_input_tokens_seen': 559939584, 'completed': '7.24% (267 / 3_687)', 'remaining time': '30:30:21', 'throughput': '8821.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:45:16,401 >> {'loss': 0.5648, 'grad_norm': 20.395606994628906, 'learning_rate': 9.987531749726137e-07, 'epoch': 0.07268782207756984, 'num_input_tokens_seen': 562036736, 'completed': '7.27% (268 / 3_687)', 'remaining time': '30:29:33', 'throughput': '8485.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:45:47,790 >> {'loss': 0.6025, 'grad_norm': 16.41619873046875, 'learning_rate': 9.98722964311952e-07, 'epoch': 0.07295904529427719, 'num_input_tokens_seen': 564133888, 'completed': '7.30% (269 / 3_687)', 'remaining time': '30:28:52', 'throughput': '8351.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:46:19,398 >> {'loss': 0.6662, 'grad_norm': 15.053420066833496, 'learning_rate': 9.986923925361238e-07, 'epoch': 0.07323026851098453, 'num_input_tokens_seen': 566231040, 'completed': '7.32% (270 / 3_687)', 'remaining time': '30:28:13', 'throughput': '8293.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:46:49,282 >> {'loss': 0.7407, 'grad_norm': 19.180273056030273, 'learning_rate': 9.98661459669732e-07, 'epoch': 0.0735014917276919, 'num_input_tokens_seen': 568328192, 'completed': '7.35% (271 / 3_687)', 'remaining time': '30:27:13', 'throughput': '8772.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:47:19,575 >> {'loss': 0.7506, 'grad_norm': 19.252796173095703, 'learning_rate': 9.986301657376705e-07, 'epoch': 0.07377271494439924, 'num_input_tokens_seen': 570425344, 'completed': '7.38% (272 / 3_687)', 'remaining time': '30:26:19', 'throughput': '8653.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:47:50,767 >> {'loss': 0.418, 'grad_norm': 12.588926315307617, 'learning_rate': 9.985985107651231e-07, 'epoch': 0.07404393816110659, 'num_input_tokens_seen': 572522496, 'completed': '7.40% (273 / 3_687)', 'remaining time': '30:25:35', 'throughput': '8404.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:48:21,534 >> {'loss': 0.9066, 'grad_norm': 19.830280303955078, 'learning_rate': 9.985664947775649e-07, 'epoch': 0.07431516137781394, 'num_input_tokens_seen': 574619648, 'completed': '7.43% (274 / 3_687)', 'remaining time': '30:24:47', 'throughput': '8520.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:48:51,039 >> {'loss': 0.5215, 'grad_norm': 12.818648338317871, 'learning_rate': 9.985341178007608e-07, 'epoch': 0.07458638459452128, 'num_input_tokens_seen': 576716800, 'completed': '7.46% (275 / 3_687)', 'remaining time': '30:23:43', 'throughput': '8884.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:49:19,349 >> {'loss': 0.8381, 'grad_norm': 18.200252532958984, 'learning_rate': 9.985013798607666e-07, 'epoch': 0.07485760781122865, 'num_input_tokens_seen': 578813952, 'completed': '7.49% (276 / 3_687)', 'remaining time': '30:22:24', 'throughput': '9259.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:49:51,985 >> {'loss': 0.6161, 'grad_norm': 17.148433685302734, 'learning_rate': 9.98468280983929e-07, 'epoch': 0.075128831027936, 'num_input_tokens_seen': 580911104, 'completed': '7.51% (277 / 3_687)', 'remaining time': '30:21:59', 'throughput': '8032.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:50:20,655 >> {'loss': 0.8064, 'grad_norm': 22.672407150268555, 'learning_rate': 9.984348211968837e-07, 'epoch': 0.07540005424464334, 'num_input_tokens_seen': 583008256, 'completed': '7.54% (278 / 3_687)', 'remaining time': '30:20:46', 'throughput': '9143.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:50:51,070 >> {'loss': 0.6633, 'grad_norm': 17.892045974731445, 'learning_rate': 9.984010005265592e-07, 'epoch': 0.07567127746135069, 'num_input_tokens_seen': 585105408, 'completed': '7.57% (279 / 3_687)', 'remaining time': '30:19:54', 'throughput': '8618.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:51:25,302 >> {'loss': 0.6054, 'grad_norm': 16.25893211364746, 'learning_rate': 9.98366819000172e-07, 'epoch': 0.07594250067805804, 'num_input_tokens_seen': 587202560, 'completed': '7.59% (280 / 3_687)', 'remaining time': '30:19:48', 'throughput': '7657.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:51:58,676 >> {'loss': 0.4507, 'grad_norm': 13.858470916748047, 'learning_rate': 9.983322766452305e-07, 'epoch': 0.0762137238947654, 'num_input_tokens_seen': 589299712, 'completed': '7.62% (281 / 3_687)', 'remaining time': '30:19:32', 'throughput': '7854.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:52:28,737 >> {'loss': 0.3055, 'grad_norm': 12.110980033874512, 'learning_rate': 9.98297373489533e-07, 'epoch': 0.07648494711147275, 'num_input_tokens_seen': 591396864, 'completed': '7.65% (282 / 3_687)', 'remaining time': '30:18:36', 'throughput': '8720.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:52:56,505 >> {'loss': 0.6864, 'grad_norm': 18.724361419677734, 'learning_rate': 9.982621095611686e-07, 'epoch': 0.07675617032818009, 'num_input_tokens_seen': 593494016, 'completed': '7.68% (283 / 3_687)', 'remaining time': '30:17:13', 'throughput': '9440.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:53:29,135 >> {'loss': 0.4906, 'grad_norm': 13.076481819152832, 'learning_rate': 9.98226484888516e-07, 'epoch': 0.07702739354488744, 'num_input_tokens_seen': 595591168, 'completed': '7.70% (284 / 3_687)', 'remaining time': '30:16:48', 'throughput': '8033.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:53:58,481 >> {'loss': 0.447, 'grad_norm': 22.247615814208984, 'learning_rate': 9.981904995002443e-07, 'epoch': 0.07729861676159479, 'num_input_tokens_seen': 597688320, 'completed': '7.73% (285 / 3_687)', 'remaining time': '30:15:44', 'throughput': '8932.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:54:29,457 >> {'loss': 0.5142, 'grad_norm': 15.656414031982422, 'learning_rate': 9.98154153425314e-07, 'epoch': 0.07756983997830215, 'num_input_tokens_seen': 599785472, 'completed': '7.76% (286 / 3_687)', 'remaining time': '30:14:59', 'throughput': '8462.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:55:01,149 >> {'loss': 0.6084, 'grad_norm': 16.987443923950195, 'learning_rate': 9.981174466929742e-07, 'epoch': 0.0778410631950095, 'num_input_tokens_seen': 601882624, 'completed': '7.78% (287 / 3_687)', 'remaining time': '30:14:23', 'throughput': '8271.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:55:32,140 >> {'loss': 0.5127, 'grad_norm': 15.571359634399414, 'learning_rate': 9.980803793327655e-07, 'epoch': 0.07811228641171684, 'num_input_tokens_seen': 603979776, 'completed': '7.81% (288 / 3_687)', 'remaining time': '30:13:39', 'throughput': '8458.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:55:59,413 >> {'loss': 0.8033, 'grad_norm': 17.951066970825195, 'learning_rate': 9.980429513745182e-07, 'epoch': 0.07838350962842419, 'num_input_tokens_seen': 606076928, 'completed': '7.84% (289 / 3_687)', 'remaining time': '30:12:12', 'throughput': '9611.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:56:30,570 >> {'loss': 1.0212, 'grad_norm': 24.38446617126465, 'learning_rate': 9.980051628483532e-07, 'epoch': 0.07865473284513154, 'num_input_tokens_seen': 608174080, 'completed': '7.87% (290 / 3_687)', 'remaining time': '30:11:30', 'throughput': '8413.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:57:00,802 >> {'loss': 0.6336, 'grad_norm': 14.629419326782227, 'learning_rate': 9.979670137846806e-07, 'epoch': 0.07892595606183889, 'num_input_tokens_seen': 610271232, 'completed': '7.89% (291 / 3_687)', 'remaining time': '30:10:37', 'throughput': '8671.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:57:30,071 >> {'loss': 1.1455, 'grad_norm': 23.52570343017578, 'learning_rate': 9.97928504214202e-07, 'epoch': 0.07919717927854625, 'num_input_tokens_seen': 612368384, 'completed': '7.92% (292 / 3_687)', 'remaining time': '30:09:33', 'throughput': '8956.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:57:55,751 >> {'loss': 0.9892, 'grad_norm': 20.963306427001953, 'learning_rate': 9.97889634167908e-07, 'epoch': 0.0794684024952536, 'num_input_tokens_seen': 614465536, 'completed': '7.95% (293 / 3_687)', 'remaining time': '30:07:48', 'throughput': '10207.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:58:26,343 >> {'loss': 0.3016, 'grad_norm': 16.687395095825195, 'learning_rate': 9.978504036770802e-07, 'epoch': 0.07973962571196094, 'num_input_tokens_seen': 616562688, 'completed': '7.97% (294 / 3_687)', 'remaining time': '30:07:01', 'throughput': '8569.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:58:58,158 >> {'loss': 0.378, 'grad_norm': 11.436803817749023, 'learning_rate': 9.978108127732892e-07, 'epoch': 0.08001084892866829, 'num_input_tokens_seen': 618659840, 'completed': '8.00% (295 / 3_687)', 'remaining time': '30:06:27', 'throughput': '8239.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 13:59:27,425 >> {'loss': 0.5094, 'grad_norm': 16.03774642944336, 'learning_rate': 9.977708614883965e-07, 'epoch': 0.08028207214537564, 'num_input_tokens_seen': 620756992, 'completed': '8.03% (296 / 3_687)', 'remaining time': '30:05:24', 'throughput': '8956.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:00:01,339 >> {'loss': 0.7122, 'grad_norm': 25.998807907104492, 'learning_rate': 9.977305498545537e-07, 'epoch': 0.080553295362083, 'num_input_tokens_seen': 622854144, 'completed': '8.06% (297 / 3_687)', 'remaining time': '30:05:15', 'throughput': '7729.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:00:30,553 >> {'loss': 0.455, 'grad_norm': 12.087435722351074, 'learning_rate': 9.976898779042018e-07, 'epoch': 0.08082451857879035, 'num_input_tokens_seen': 624951296, 'completed': '8.08% (298 / 3_687)', 'remaining time': '30:04:12', 'throughput': '8973.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:01:03,007 >> {'loss': 0.7966, 'grad_norm': 19.255043029785156, 'learning_rate': 9.976488456700717e-07, 'epoch': 0.0810957417954977, 'num_input_tokens_seen': 627048448, 'completed': '8.11% (299 / 3_687)', 'remaining time': '30:03:46', 'throughput': '8077.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:01:30,569 >> {'loss': 0.61, 'grad_norm': 15.657422065734863, 'learning_rate': 9.97607453185185e-07, 'epoch': 0.08136696501220504, 'num_input_tokens_seen': 629145600, 'completed': '8.14% (300 / 3_687)', 'remaining time': '30:02:24', 'throughput': '9511.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:02:02,816 >> {'loss': 0.7554, 'grad_norm': 17.275453567504883, 'learning_rate': 9.97565700482853e-07, 'epoch': 0.08163818822891239, 'num_input_tokens_seen': 631242752, 'completed': '8.16% (301 / 3_687)', 'remaining time': '30:01:56', 'throughput': '8129.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:02:34,217 >> {'loss': 0.5496, 'grad_norm': 16.451929092407227, 'learning_rate': 9.97523587596676e-07, 'epoch': 0.08190941144561975, 'num_input_tokens_seen': 633339904, 'completed': '8.19% (302 / 3_687)', 'remaining time': '30:01:18', 'throughput': '8348.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:03:03,565 >> {'loss': 0.2887, 'grad_norm': 11.221899032592773, 'learning_rate': 9.974811145605453e-07, 'epoch': 0.0821806346623271, 'num_input_tokens_seen': 635437056, 'completed': '8.22% (303 / 3_687)', 'remaining time': '30:00:17', 'throughput': '8932.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:03:33,684 >> {'loss': 0.3712, 'grad_norm': 15.957961082458496, 'learning_rate': 9.974382814086418e-07, 'epoch': 0.08245185787903445, 'num_input_tokens_seen': 637534208, 'completed': '8.25% (304 / 3_687)', 'remaining time': '29:59:25', 'throughput': '8703.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:04:03,806 >> {'loss': 0.4835, 'grad_norm': 14.90071964263916, 'learning_rate': 9.973950881754353e-07, 'epoch': 0.0827230810957418, 'num_input_tokens_seen': 639631360, 'completed': '8.27% (305 / 3_687)', 'remaining time': '29:58:34', 'throughput': '8702.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:04:31,205 >> {'loss': 0.6427, 'grad_norm': 17.702465057373047, 'learning_rate': 9.973515348956869e-07, 'epoch': 0.08299430431244914, 'num_input_tokens_seen': 641728512, 'completed': '8.30% (306 / 3_687)', 'remaining time': '29:57:12', 'throughput': '9567.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:05:01,747 >> {'loss': 0.5525, 'grad_norm': 13.002301216125488, 'learning_rate': 9.97307621604446e-07, 'epoch': 0.0832655275291565, 'num_input_tokens_seen': 643825664, 'completed': '8.33% (307 / 3_687)', 'remaining time': '29:56:25', 'throughput': '8583.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:05:31,788 >> {'loss': 0.4219, 'grad_norm': 12.444464683532715, 'learning_rate': 9.972633483370526e-07, 'epoch': 0.08353675074586385, 'num_input_tokens_seen': 645922816, 'completed': '8.35% (308 / 3_687)', 'remaining time': '29:55:33', 'throughput': '8726.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:06:01,351 >> {'loss': 0.8305, 'grad_norm': 34.6700439453125, 'learning_rate': 9.97218715129136e-07, 'epoch': 0.0838079739625712, 'num_input_tokens_seen': 648019968, 'completed': '8.38% (309 / 3_687)', 'remaining time': '29:54:36', 'throughput': '8867.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:06:30,838 >> {'loss': 0.4636, 'grad_norm': 13.552191734313965, 'learning_rate': 9.971737220166155e-07, 'epoch': 0.08407919717927854, 'num_input_tokens_seen': 650117120, 'completed': '8.41% (310 / 3_687)', 'remaining time': '29:53:38', 'throughput': '8890.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:07:01,651 >> {'loss': 0.68, 'grad_norm': 15.92861270904541, 'learning_rate': 9.971283690356997e-07, 'epoch': 0.08435042039598589, 'num_input_tokens_seen': 652214272, 'completed': '8.44% (311 / 3_687)', 'remaining time': '29:52:54', 'throughput': '8507.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:07:33,097 >> {'loss': 0.292, 'grad_norm': 10.955565452575684, 'learning_rate': 9.97082656222887e-07, 'epoch': 0.08462164361269324, 'num_input_tokens_seen': 654311424, 'completed': '8.46% (312 / 3_687)', 'remaining time': '29:52:18', 'throughput': '8336.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:08:03,034 >> {'loss': 0.6708, 'grad_norm': 16.476566314697266, 'learning_rate': 9.970365836149654e-07, 'epoch': 0.0848928668294006, 'num_input_tokens_seen': 656408576, 'completed': '8.49% (313 / 3_687)', 'remaining time': '29:51:25', 'throughput': '8756.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:08:32,278 >> {'loss': 1.0652, 'grad_norm': 21.01806640625, 'learning_rate': 9.969901512490121e-07, 'epoch': 0.08516409004610795, 'num_input_tokens_seen': 658505728, 'completed': '8.52% (314 / 3_687)', 'remaining time': '29:50:26', 'throughput': '8964.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:09:01,930 >> {'loss': 0.6264, 'grad_norm': 16.38351058959961, 'learning_rate': 9.969433591623946e-07, 'epoch': 0.0854353132628153, 'num_input_tokens_seen': 660602880, 'completed': '8.54% (315 / 3_687)', 'remaining time': '29:49:30', 'throughput': '8840.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:09:30,441 >> {'loss': 1.0512, 'grad_norm': 23.092439651489258, 'learning_rate': 9.96896207392769e-07, 'epoch': 0.08570653647952264, 'num_input_tokens_seen': 662700032, 'completed': '8.57% (316 / 3_687)', 'remaining time': '29:48:23', 'throughput': '9194.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:10:04,106 >> {'loss': 0.5465, 'grad_norm': 14.13205337524414, 'learning_rate': 9.968486959780813e-07, 'epoch': 0.08597775969622999, 'num_input_tokens_seen': 664797184, 'completed': '8.60% (317 / 3_687)', 'remaining time': '29:48:10', 'throughput': '7786.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:10:32,766 >> {'loss': 0.6146, 'grad_norm': 16.44294548034668, 'learning_rate': 9.96800824956567e-07, 'epoch': 0.08624898291293735, 'num_input_tokens_seen': 666894336, 'completed': '8.62% (318 / 3_687)', 'remaining time': '29:47:05', 'throughput': '9146.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:11:03,754 >> {'loss': 0.6067, 'grad_norm': 16.18340301513672, 'learning_rate': 9.967525943667506e-07, 'epoch': 0.0865202061296447, 'num_input_tokens_seen': 668991488, 'completed': '8.65% (319 / 3_687)', 'remaining time': '29:46:24', 'throughput': '8459.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:11:33,706 >> {'loss': 0.5758, 'grad_norm': 14.490288734436035, 'learning_rate': 9.967040042474467e-07, 'epoch': 0.08679142934635205, 'num_input_tokens_seen': 671088640, 'completed': '8.68% (320 / 3_687)', 'remaining time': '29:45:33', 'throughput': '8752.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:12:05,369 >> {'loss': 0.8391, 'grad_norm': 20.6240234375, 'learning_rate': 9.966550546377586e-07, 'epoch': 0.0870626525630594, 'num_input_tokens_seen': 673185792, 'completed': '8.71% (321 / 3_687)', 'remaining time': '29:44:59', 'throughput': '8279.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:12:35,289 >> {'loss': 0.3815, 'grad_norm': 12.437811851501465, 'learning_rate': 9.966057455770788e-07, 'epoch': 0.08733387577976674, 'num_input_tokens_seen': 675282944, 'completed': '8.73% (322 / 3_687)', 'remaining time': '29:44:08', 'throughput': '8761.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:13:05,607 >> {'loss': 0.7666, 'grad_norm': 17.535430908203125, 'learning_rate': 9.965560771050896e-07, 'epoch': 0.0876050989964741, 'num_input_tokens_seen': 677380096, 'completed': '8.76% (323 / 3_687)', 'remaining time': '29:43:20', 'throughput': '8646.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:13:37,330 >> {'loss': 0.6267, 'grad_norm': 17.357746124267578, 'learning_rate': 9.965060492617623e-07, 'epoch': 0.08787632221318145, 'num_input_tokens_seen': 679477248, 'completed': '8.79% (324 / 3_687)', 'remaining time': '29:42:48', 'throughput': '8263.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:14:09,267 >> {'loss': 0.6236, 'grad_norm': 19.566974639892578, 'learning_rate': 9.964556620873573e-07, 'epoch': 0.0881475454298888, 'num_input_tokens_seen': 681574400, 'completed': '8.81% (325 / 3_687)', 'remaining time': '29:42:17', 'throughput': '8208.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:14:37,896 >> {'loss': 0.5585, 'grad_norm': 14.313620567321777, 'learning_rate': 9.964049156224244e-07, 'epoch': 0.08841876864659615, 'num_input_tokens_seen': 683671552, 'completed': '8.84% (326 / 3_687)', 'remaining time': '29:41:13', 'throughput': '9156.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:15:10,739 >> {'loss': 0.4307, 'grad_norm': 13.445440292358398, 'learning_rate': 9.963538099078024e-07, 'epoch': 0.0886899918633035, 'num_input_tokens_seen': 685768704, 'completed': '8.87% (327 / 3_687)', 'remaining time': '29:40:51', 'throughput': '7981.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:15:43,730 >> {'loss': 0.7462, 'grad_norm': 17.229440689086914, 'learning_rate': 9.963023449846194e-07, 'epoch': 0.08896121508001086, 'num_input_tokens_seen': 687865856, 'completed': '8.90% (328 / 3_687)', 'remaining time': '29:40:32', 'throughput': '7945.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:16:14,781 >> {'loss': 0.9177, 'grad_norm': 20.201749801635742, 'learning_rate': 9.962505208942919e-07, 'epoch': 0.0892324382967182, 'num_input_tokens_seen': 689963008, 'completed': '8.92% (329 / 3_687)', 'remaining time': '29:39:52', 'throughput': '8442.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:16:46,829 >> {'loss': 0.4933, 'grad_norm': 13.015423774719238, 'learning_rate': 9.961983376785264e-07, 'epoch': 0.08950366151342555, 'num_input_tokens_seen': 692060160, 'completed': '8.95% (330 / 3_687)', 'remaining time': '29:39:23', 'throughput': '8179.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:17:19,438 >> {'loss': 0.3818, 'grad_norm': 11.634929656982422, 'learning_rate': 9.96145795379318e-07, 'epoch': 0.0897748847301329, 'num_input_tokens_seen': 694157312, 'completed': '8.98% (331 / 3_687)', 'remaining time': '29:38:59', 'throughput': '8039.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:17:48,482 >> {'loss': 0.8161, 'grad_norm': 18.479869842529297, 'learning_rate': 9.960928940389503e-07, 'epoch': 0.09004610794684025, 'num_input_tokens_seen': 696254464, 'completed': '9.00% (332 / 3_687)', 'remaining time': '29:38:00', 'throughput': '9025.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:18:23,318 >> {'loss': 0.451, 'grad_norm': 12.271891593933105, 'learning_rate': 9.960396336999967e-07, 'epoch': 0.09031733116354759, 'num_input_tokens_seen': 698351616, 'completed': '9.03% (333 / 3_687)', 'remaining time': '29:37:59', 'throughput': '7525.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:18:53,792 >> {'loss': 0.3789, 'grad_norm': 10.953863143920898, 'learning_rate': 9.95986014405319e-07, 'epoch': 0.09058855438025495, 'num_input_tokens_seen': 700448768, 'completed': '9.06% (334 / 3_687)', 'remaining time': '29:37:13', 'throughput': '8602.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:19:23,689 >> {'loss': 0.3936, 'grad_norm': 9.575960159301758, 'learning_rate': 9.959320361980679e-07, 'epoch': 0.0908597775969623, 'num_input_tokens_seen': 702545920, 'completed': '9.09% (335 / 3_687)', 'remaining time': '29:36:22', 'throughput': '8768.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:19:54,663 >> {'loss': 0.256, 'grad_norm': 13.222553253173828, 'learning_rate': 9.95877699121683e-07, 'epoch': 0.09113100081366965, 'num_input_tokens_seen': 704643072, 'completed': '9.11% (336 / 3_687)', 'remaining time': '29:35:42', 'throughput': '8463.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:20:24,844 >> {'loss': 1.0641, 'grad_norm': 20.804859161376953, 'learning_rate': 9.95823003219893e-07, 'epoch': 0.091402224030377, 'num_input_tokens_seen': 706740224, 'completed': '9.14% (337 / 3_687)', 'remaining time': '29:34:55', 'throughput': '8685.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:20:58,559 >> {'loss': 0.3291, 'grad_norm': 11.381640434265137, 'learning_rate': 9.957679485367144e-07, 'epoch': 0.09167344724708434, 'num_input_tokens_seen': 708837376, 'completed': '9.17% (338 / 3_687)', 'remaining time': '29:34:42', 'throughput': '7775.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:21:29,539 >> {'loss': 0.4397, 'grad_norm': 13.033456802368164, 'learning_rate': 9.95712535116454e-07, 'epoch': 0.0919446704637917, 'num_input_tokens_seen': 710934528, 'completed': '9.19% (339 / 3_687)', 'remaining time': '29:34:02', 'throughput': '8461.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:22:01,722 >> {'loss': 0.6069, 'grad_norm': 21.973115921020508, 'learning_rate': 9.956567630037058e-07, 'epoch': 0.09221589368049905, 'num_input_tokens_seen': 713031680, 'completed': '9.22% (340 / 3_687)', 'remaining time': '29:33:34', 'throughput': '8145.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:22:30,626 >> {'loss': 0.8007, 'grad_norm': 20.099735260009766, 'learning_rate': 9.95600632243353e-07, 'epoch': 0.0924871168972064, 'num_input_tokens_seen': 715128832, 'completed': '9.25% (341 / 3_687)', 'remaining time': '29:32:34', 'throughput': '9069.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:23:03,136 >> {'loss': 0.7435, 'grad_norm': 19.394807815551758, 'learning_rate': 9.95544142880568e-07, 'epoch': 0.09275834011391375, 'num_input_tokens_seen': 717225984, 'completed': '9.28% (342 / 3_687)', 'remaining time': '29:32:09', 'throughput': '8063.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:23:34,167 >> {'loss': 0.5055, 'grad_norm': 17.3151798248291, 'learning_rate': 9.954872949608108e-07, 'epoch': 0.0930295633306211, 'num_input_tokens_seen': 719323136, 'completed': '9.30% (343 / 3_687)', 'remaining time': '29:31:30', 'throughput': '8447.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:24:01,961 >> {'loss': 0.6907, 'grad_norm': 19.800769805908203, 'learning_rate': 9.954300885298309e-07, 'epoch': 0.09330078654732846, 'num_input_tokens_seen': 721420288, 'completed': '9.33% (344 / 3_687)', 'remaining time': '29:30:20', 'throughput': '9431.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:24:31,792 >> {'loss': 0.3009, 'grad_norm': 11.735955238342285, 'learning_rate': 9.953725236336653e-07, 'epoch': 0.0935720097640358, 'num_input_tokens_seen': 723517440, 'completed': '9.36% (345 / 3_687)', 'remaining time': '29:29:29', 'throughput': '8787.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:25:03,219 >> {'loss': 0.577, 'grad_norm': 15.41151237487793, 'learning_rate': 9.953146003186407e-07, 'epoch': 0.09384323298074315, 'num_input_tokens_seen': 725614592, 'completed': '9.38% (346 / 3_687)', 'remaining time': '29:28:54', 'throughput': '8341.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:25:35,122 >> {'loss': 0.9646, 'grad_norm': 21.002737045288086, 'learning_rate': 9.952563186313711e-07, 'epoch': 0.0941144561974505, 'num_input_tokens_seen': 727711744, 'completed': '9.41% (347 / 3_687)', 'remaining time': '29:28:23', 'throughput': '8217.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:26:08,226 >> {'loss': 0.1761, 'grad_norm': 8.514850616455078, 'learning_rate': 9.951976786187598e-07, 'epoch': 0.09438567941415785, 'num_input_tokens_seen': 729808896, 'completed': '9.44% (348 / 3_687)', 'remaining time': '29:28:04', 'throughput': '7918.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:26:43,324 >> {'loss': 0.5064, 'grad_norm': 17.795473098754883, 'learning_rate': 9.951386803279973e-07, 'epoch': 0.09465690263086521, 'num_input_tokens_seen': 731906048, 'completed': '9.47% (349 / 3_687)', 'remaining time': '29:28:05', 'throughput': '7468.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:27:14,092 >> {'loss': 0.5842, 'grad_norm': 15.562695503234863, 'learning_rate': 9.95079323806564e-07, 'epoch': 0.09492812584757256, 'num_input_tokens_seen': 734003200, 'completed': '9.49% (350 / 3_687)', 'remaining time': '29:27:23', 'throughput': '8519.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:27:46,442 >> {'loss': 0.4142, 'grad_norm': 14.850616455078125, 'learning_rate': 9.950196091022274e-07, 'epoch': 0.0951993490642799, 'num_input_tokens_seen': 736100352, 'completed': '9.52% (351 / 3_687)', 'remaining time': '29:26:57', 'throughput': '8103.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:28:15,446 >> {'loss': 0.3965, 'grad_norm': 13.248444557189941, 'learning_rate': 9.949595362630435e-07, 'epoch': 0.09547057228098725, 'num_input_tokens_seen': 738197504, 'completed': '9.55% (352 / 3_687)', 'remaining time': '29:25:59', 'throughput': '9038.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:28:43,623 >> {'loss': 0.6689, 'grad_norm': 19.31031608581543, 'learning_rate': 9.948991053373567e-07, 'epoch': 0.0957417954976946, 'num_input_tokens_seen': 740294656, 'completed': '9.57% (353 / 3_687)', 'remaining time': '29:24:53', 'throughput': '9303.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:29:15,925 >> {'loss': 0.6139, 'grad_norm': 18.403900146484375, 'learning_rate': 9.948383163738e-07, 'epoch': 0.09601301871440195, 'num_input_tokens_seen': 742391808, 'completed': '9.60% (354 / 3_687)', 'remaining time': '29:24:26', 'throughput': '8115.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:29:48,593 >> {'loss': 0.5709, 'grad_norm': 16.723413467407227, 'learning_rate': 9.947771694212933e-07, 'epoch': 0.09628424193110931, 'num_input_tokens_seen': 744488960, 'completed': '9.63% (355 / 3_687)', 'remaining time': '29:24:03', 'throughput': '8024.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:30:19,100 >> {'loss': 0.6557, 'grad_norm': 14.325742721557617, 'learning_rate': 9.947156645290456e-07, 'epoch': 0.09655546514781665, 'num_input_tokens_seen': 746586112, 'completed': '9.66% (356 / 3_687)', 'remaining time': '29:23:19', 'throughput': '8593.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:30:53,114 >> {'loss': 0.8035, 'grad_norm': 19.636194229125977, 'learning_rate': 9.94653801746554e-07, 'epoch': 0.096826688364524, 'num_input_tokens_seen': 748683264, 'completed': '9.68% (357 / 3_687)', 'remaining time': '29:23:09', 'throughput': '7706.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:31:22,506 >> {'loss': 0.508, 'grad_norm': 13.98243236541748, 'learning_rate': 9.945915811236029e-07, 'epoch': 0.09709791158123135, 'num_input_tokens_seen': 750780416, 'completed': '9.71% (358 / 3_687)', 'remaining time': '29:22:15', 'throughput': '8918.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:31:53,149 >> {'loss': 0.4487, 'grad_norm': 14.497775077819824, 'learning_rate': 9.945290027102654e-07, 'epoch': 0.0973691347979387, 'num_input_tokens_seen': 752877568, 'completed': '9.74% (359 / 3_687)', 'remaining time': '29:21:33', 'throughput': '8554.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:32:21,892 >> {'loss': 0.6489, 'grad_norm': 15.805448532104492, 'learning_rate': 9.944660665569023e-07, 'epoch': 0.09764035801464606, 'num_input_tokens_seen': 754974720, 'completed': '9.76% (360 / 3_687)', 'remaining time': '29:20:33', 'throughput': '9120.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:32:49,298 >> {'loss': 0.5395, 'grad_norm': 14.843584060668945, 'learning_rate': 9.944027727141617e-07, 'epoch': 0.0979115812313534, 'num_input_tokens_seen': 757071872, 'completed': '9.79% (361 / 3_687)', 'remaining time': '29:19:21', 'throughput': '9565.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:33:21,416 >> {'loss': 0.4855, 'grad_norm': 15.000771522521973, 'learning_rate': 9.943391212329805e-07, 'epoch': 0.09818280444806075, 'num_input_tokens_seen': 759169024, 'completed': '9.82% (362 / 3_687)', 'remaining time': '29:18:53', 'throughput': '8161.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:33:49,226 >> {'loss': 1.1353, 'grad_norm': 26.341798782348633, 'learning_rate': 9.942751121645828e-07, 'epoch': 0.0984540276647681, 'num_input_tokens_seen': 761266176, 'completed': '9.85% (363 / 3_687)', 'remaining time': '29:17:45', 'throughput': '9426.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:34:22,516 >> {'loss': 0.5821, 'grad_norm': 14.345602989196777, 'learning_rate': 9.94210745560481e-07, 'epoch': 0.09872525088147545, 'num_input_tokens_seen': 763363328, 'completed': '9.87% (364 / 3_687)', 'remaining time': '29:17:28', 'throughput': '7874.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:34:51,868 >> {'loss': 0.3652, 'grad_norm': 13.088194847106934, 'learning_rate': 9.941460214724747e-07, 'epoch': 0.09899647409818281, 'num_input_tokens_seen': 765460480, 'completed': '9.90% (365 / 3_687)', 'remaining time': '29:16:34', 'throughput': '8931.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:35:21,617 >> {'loss': 0.8981, 'grad_norm': 25.987003326416016, 'learning_rate': 9.94080939952651e-07, 'epoch': 0.09926769731489016, 'num_input_tokens_seen': 767557632, 'completed': '9.93% (366 / 3_687)', 'remaining time': '29:15:45', 'throughput': '8812.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:35:49,113 >> {'loss': 0.721, 'grad_norm': 19.704673767089844, 'learning_rate': 9.940155010533855e-07, 'epoch': 0.0995389205315975, 'num_input_tokens_seen': 769654784, 'completed': '9.95% (367 / 3_687)', 'remaining time': '29:14:35', 'throughput': '9533.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:36:20,565 >> {'loss': 0.741, 'grad_norm': 15.577407836914062, 'learning_rate': 9.939497048273407e-07, 'epoch': 0.09981014374830485, 'num_input_tokens_seen': 771751936, 'completed': '9.98% (368 / 3_687)', 'remaining time': '29:14:01', 'throughput': '8334.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:36:50,662 >> {'loss': 0.7845, 'grad_norm': 23.83604621887207, 'learning_rate': 9.938835513274672e-07, 'epoch': 0.1000813669650122, 'num_input_tokens_seen': 773849088, 'completed': '10.01% (369 / 3_687)', 'remaining time': '29:13:15', 'throughput': '8709.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:37:19,740 >> {'loss': 0.4799, 'grad_norm': 13.335168838500977, 'learning_rate': 9.938170406070025e-07, 'epoch': 0.10035259018171956, 'num_input_tokens_seen': 775946240, 'completed': '10.04% (370 / 3_687)', 'remaining time': '29:12:19', 'throughput': '9015.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:37:52,342 >> {'loss': 0.4814, 'grad_norm': 12.84589672088623, 'learning_rate': 9.937501727194721e-07, 'epoch': 0.10062381339842691, 'num_input_tokens_seen': 778043392, 'completed': '10.06% (371 / 3_687)', 'remaining time': '29:11:56', 'throughput': '8040.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:38:23,007 >> {'loss': 0.6229, 'grad_norm': 17.722841262817383, 'learning_rate': 9.936829477186884e-07, 'epoch': 0.10089503661513426, 'num_input_tokens_seen': 780140544, 'completed': '10.09% (372 / 3_687)', 'remaining time': '29:11:15', 'throughput': '8548.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:38:54,234 >> {'loss': 0.3687, 'grad_norm': 11.619221687316895, 'learning_rate': 9.93615365658752e-07, 'epoch': 0.1011662598318416, 'num_input_tokens_seen': 782237696, 'completed': '10.12% (373 / 3_687)', 'remaining time': '29:10:39', 'throughput': '8394.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:39:23,874 >> {'loss': 0.7215, 'grad_norm': 17.260656356811523, 'learning_rate': 9.9354742659405e-07, 'epoch': 0.10143748304854895, 'num_input_tokens_seen': 784334848, 'completed': '10.14% (374 / 3_687)', 'remaining time': '29:09:49', 'throughput': '8844.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:39:55,358 >> {'loss': 1.015, 'grad_norm': 19.690866470336914, 'learning_rate': 9.934791305792575e-07, 'epoch': 0.10170870626525631, 'num_input_tokens_seen': 786432000, 'completed': '10.17% (375 / 3_687)', 'remaining time': '29:09:16', 'throughput': '8326.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:40:21,822 >> {'loss': 1.135, 'grad_norm': 22.19734001159668, 'learning_rate': 9.934104776693363e-07, 'epoch': 0.10197992948196366, 'num_input_tokens_seen': 788529152, 'completed': '10.20% (376 / 3_687)', 'remaining time': '29:07:58', 'throughput': '9905.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:40:51,365 >> {'loss': 0.4608, 'grad_norm': 12.495186805725098, 'learning_rate': 9.933414679195354e-07, 'epoch': 0.10225115269867101, 'num_input_tokens_seen': 790626304, 'completed': '10.23% (377 / 3_687)', 'remaining time': '29:07:07', 'throughput': '8873.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:41:24,785 >> {'loss': 0.4727, 'grad_norm': 13.882384300231934, 'learning_rate': 9.932721013853917e-07, 'epoch': 0.10252237591537836, 'num_input_tokens_seen': 792723456, 'completed': '10.25% (378 / 3_687)', 'remaining time': '29:06:51', 'throughput': '7844.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:41:56,723 >> {'loss': 0.3467, 'grad_norm': 17.025941848754883, 'learning_rate': 9.932023781227287e-07, 'epoch': 0.1027935991320857, 'num_input_tokens_seen': 794820608, 'completed': '10.28% (379 / 3_687)', 'remaining time': '29:06:22', 'throughput': '8209.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:42:27,798 >> {'loss': 0.3002, 'grad_norm': 9.603739738464355, 'learning_rate': 9.931322981876567e-07, 'epoch': 0.10306482234879305, 'num_input_tokens_seen': 796917760, 'completed': '10.31% (380 / 3_687)', 'remaining time': '29:05:45', 'throughput': '8434.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:43:00,949 >> {'loss': 0.2998, 'grad_norm': 9.856173515319824, 'learning_rate': 9.930618616365737e-07, 'epoch': 0.10333604556550041, 'num_input_tokens_seen': 799014912, 'completed': '10.33% (381 / 3_687)', 'remaining time': '29:05:26', 'throughput': '7907.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:43:31,176 >> {'loss': 0.6395, 'grad_norm': 16.00568199157715, 'learning_rate': 9.92991068526164e-07, 'epoch': 0.10360726878220776, 'num_input_tokens_seen': 801112064, 'completed': '10.36% (382 / 3_687)', 'remaining time': '29:04:42', 'throughput': '8672.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:44:00,631 >> {'loss': 0.4683, 'grad_norm': 13.31304931640625, 'learning_rate': 9.929199189133996e-07, 'epoch': 0.1038784919989151, 'num_input_tokens_seen': 803209216, 'completed': '10.39% (383 / 3_687)', 'remaining time': '29:03:51', 'throughput': '8899.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:44:26,163 >> {'loss': 0.9509, 'grad_norm': 22.845258712768555, 'learning_rate': 9.928484128555388e-07, 'epoch': 0.10414971521562245, 'num_input_tokens_seen': 805306368, 'completed': '10.41% (384 / 3_687)', 'remaining time': '29:02:26', 'throughput': '10267.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:44:59,390 >> {'loss': 0.6683, 'grad_norm': 18.43478012084961, 'learning_rate': 9.92776550410127e-07, 'epoch': 0.1044209384323298, 'num_input_tokens_seen': 807403520, 'completed': '10.44% (385 / 3_687)', 'remaining time': '29:02:08', 'throughput': '7889.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:45:30,095 >> {'loss': 0.375, 'grad_norm': 19.82917022705078, 'learning_rate': 9.927043316349962e-07, 'epoch': 0.10469216164903716, 'num_input_tokens_seen': 809500672, 'completed': '10.47% (386 / 3_687)', 'remaining time': '29:01:28', 'throughput': '8537.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:46:04,246 >> {'loss': 0.4983, 'grad_norm': 17.08930206298828, 'learning_rate': 9.926317565882657e-07, 'epoch': 0.10496338486574451, 'num_input_tokens_seen': 811597824, 'completed': '10.50% (387 / 3_687)', 'remaining time': '29:01:18', 'throughput': '7675.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:46:36,601 >> {'loss': 0.433, 'grad_norm': 13.114651679992676, 'learning_rate': 9.925588253283407e-07, 'epoch': 0.10523460808245186, 'num_input_tokens_seen': 813694976, 'completed': '10.52% (388 / 3_687)', 'remaining time': '29:00:52', 'throughput': '8102.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:47:06,649 >> {'loss': 0.4286, 'grad_norm': 12.93115234375, 'learning_rate': 9.924855379139136e-07, 'epoch': 0.1055058312991592, 'num_input_tokens_seen': 815792128, 'completed': '10.55% (389 / 3_687)', 'remaining time': '29:00:07', 'throughput': '8724.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:47:38,944 >> {'loss': 0.5672, 'grad_norm': 16.018659591674805, 'learning_rate': 9.924118944039635e-07, 'epoch': 0.10577705451586655, 'num_input_tokens_seen': 817889280, 'completed': '10.58% (390 / 3_687)', 'remaining time': '28:59:41', 'throughput': '8117.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:48:10,168 >> {'loss': 0.7503, 'grad_norm': 17.725582122802734, 'learning_rate': 9.923378948577558e-07, 'epoch': 0.10604827773257391, 'num_input_tokens_seen': 819986432, 'completed': '10.60% (391 / 3_687)', 'remaining time': '28:59:05', 'throughput': '8395.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:48:42,227 >> {'loss': 0.4547, 'grad_norm': 12.33151912689209, 'learning_rate': 9.922635393348425e-07, 'epoch': 0.10631950094928126, 'num_input_tokens_seen': 822083584, 'completed': '10.63% (392 / 3_687)', 'remaining time': '28:58:37', 'throughput': '8176.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:49:13,087 >> {'loss': 0.4688, 'grad_norm': 15.34462833404541, 'learning_rate': 9.92188827895062e-07, 'epoch': 0.10659072416598861, 'num_input_tokens_seen': 824180736, 'completed': '10.66% (393 / 3_687)', 'remaining time': '28:57:59', 'throughput': '8494.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:49:43,133 >> {'loss': 0.8941, 'grad_norm': 20.61101722717285, 'learning_rate': 9.921137605985397e-07, 'epoch': 0.10686194738269596, 'num_input_tokens_seen': 826277888, 'completed': '10.69% (394 / 3_687)', 'remaining time': '28:57:14', 'throughput': '8724.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:50:15,635 >> {'loss': 0.584, 'grad_norm': 14.963687896728516, 'learning_rate': 9.920383375056863e-07, 'epoch': 0.1071331705994033, 'num_input_tokens_seen': 828375040, 'completed': '10.71% (395 / 3_687)', 'remaining time': '28:56:49', 'throughput': '8065.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:50:43,025 >> {'loss': 0.7246, 'grad_norm': 17.79062843322754, 'learning_rate': 9.919625586771998e-07, 'epoch': 0.10740439381611067, 'num_input_tokens_seen': 830472192, 'completed': '10.74% (396 / 3_687)', 'remaining time': '28:55:42', 'throughput': '9571.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:51:15,662 >> {'loss': 0.6275, 'grad_norm': 16.45888328552246, 'learning_rate': 9.918864241740639e-07, 'epoch': 0.10767561703281801, 'num_input_tokens_seen': 832569344, 'completed': '10.77% (397 / 3_687)', 'remaining time': '28:55:19', 'throughput': '8032.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:51:48,628 >> {'loss': 0.53, 'grad_norm': 17.214998245239258, 'learning_rate': 9.918099340575487e-07, 'epoch': 0.10794684024952536, 'num_input_tokens_seen': 834666496, 'completed': '10.79% (398 / 3_687)', 'remaining time': '28:54:58', 'throughput': '7951.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:52:17,810 >> {'loss': 0.5958, 'grad_norm': 18.772640228271484, 'learning_rate': 9.91733088389211e-07, 'epoch': 0.10821806346623271, 'num_input_tokens_seen': 836763648, 'completed': '10.82% (399 / 3_687)', 'remaining time': '28:54:06', 'throughput': '8983.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:52:48,368 >> {'loss': 0.3651, 'grad_norm': 13.285908699035645, 'learning_rate': 9.916558872308929e-07, 'epoch': 0.10848928668294006, 'num_input_tokens_seen': 838860800, 'completed': '10.85% (400 / 3_687)', 'remaining time': '28:53:25', 'throughput': '8578.58', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2025-01-01 14:53:12,867 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-400 +[INFO|configuration_utils.py:472] 2025-01-01 14:53:12,872 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-400/config.json +[INFO|configuration_utils.py:807] 2025-01-01 14:53:12,873 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-400/generation_config.json +[INFO|modeling_utils.py:2807] 2025-01-01 14:54:11,511 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-400/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2025-01-01 14:54:11,516 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-400/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2025-01-01 14:54:11,517 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-400/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2025-01-01 14:58:05,146 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 800, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2025-01-01 14:58:37,408 >> {'loss': 0.5324, 'grad_norm': 14.000696182250977, 'learning_rate': 9.915783306447229e-07, 'epoch': 0.1087605098996474, 'num_input_tokens_seen': 840957952, 'completed': '10.88% (401 / 3_687)', 'remaining time': '29:36:15', 'throughput': '751.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:59:06,446 >> {'loss': 0.5363, 'grad_norm': 14.921085357666016, 'learning_rate': 9.915004186931156e-07, 'epoch': 0.10903173311635476, 'num_input_tokens_seen': 843055104, 'completed': '10.90% (402 / 3_687)', 'remaining time': '29:35:14', 'throughput': '9027.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 14:59:35,798 >> {'loss': 0.3414, 'grad_norm': 13.49205493927002, 'learning_rate': 9.91422151438772e-07, 'epoch': 0.10930295633306211, 'num_input_tokens_seen': 845152256, 'completed': '10.93% (403 / 3_687)', 'remaining time': '29:34:17', 'throughput': '8931.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:00:05,863 >> {'loss': 0.8933, 'grad_norm': 20.21406364440918, 'learning_rate': 9.91343528944678e-07, 'epoch': 0.10957417954976946, 'num_input_tokens_seen': 847249408, 'completed': '10.96% (404 / 3_687)', 'remaining time': '29:33:25', 'throughput': '8719.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:00:41,024 >> {'loss': 0.3803, 'grad_norm': 11.765609741210938, 'learning_rate': 9.912645512741064e-07, 'epoch': 0.10984540276647681, 'num_input_tokens_seen': 849346560, 'completed': '10.98% (405 / 3_687)', 'remaining time': '29:33:15', 'throughput': '7455.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:01:11,475 >> {'loss': 0.5149, 'grad_norm': 22.0726318359375, 'learning_rate': 9.911852184906151e-07, 'epoch': 0.11011662598318415, 'num_input_tokens_seen': 851443712, 'completed': '11.01% (406 / 3_687)', 'remaining time': '29:32:27', 'throughput': '8608.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:01:42,302 >> {'loss': 0.5114, 'grad_norm': 14.043606758117676, 'learning_rate': 9.911055306580485e-07, 'epoch': 0.11038784919989152, 'num_input_tokens_seen': 853540864, 'completed': '11.04% (407 / 3_687)', 'remaining time': '29:31:42', 'throughput': '8503.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:02:12,017 >> {'loss': 0.3898, 'grad_norm': 12.334258079528809, 'learning_rate': 9.910254878405361e-07, 'epoch': 0.11065907241659886, 'num_input_tokens_seen': 855638016, 'completed': '11.07% (408 / 3_687)', 'remaining time': '29:30:48', 'throughput': '8822.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:02:42,181 >> {'loss': 0.3856, 'grad_norm': 11.179019927978516, 'learning_rate': 9.909450901024935e-07, 'epoch': 0.11093029563330621, 'num_input_tokens_seen': 857735168, 'completed': '11.09% (409 / 3_687)', 'remaining time': '29:29:57', 'throughput': '8690.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:03:12,030 >> {'loss': 0.7504, 'grad_norm': 17.625316619873047, 'learning_rate': 9.908643375086213e-07, 'epoch': 0.11120151885001356, 'num_input_tokens_seen': 859832320, 'completed': '11.12% (410 / 3_687)', 'remaining time': '29:29:05', 'throughput': '8782.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:03:43,468 >> {'loss': 0.4077, 'grad_norm': 11.948003768920898, 'learning_rate': 9.907832301239066e-07, 'epoch': 0.1114727420667209, 'num_input_tokens_seen': 861929472, 'completed': '11.15% (411 / 3_687)', 'remaining time': '29:28:25', 'throughput': '8338.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:04:17,221 >> {'loss': 0.8141, 'grad_norm': 22.031713485717773, 'learning_rate': 9.907017680136213e-07, 'epoch': 0.11174396528342827, 'num_input_tokens_seen': 864026624, 'completed': '11.17% (412 / 3_687)', 'remaining time': '29:28:03', 'throughput': '7766.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:04:47,842 >> {'loss': 0.5477, 'grad_norm': 14.142679214477539, 'learning_rate': 9.90619951243323e-07, 'epoch': 0.11201518850013562, 'num_input_tokens_seen': 866123776, 'completed': '11.20% (413 / 3_687)', 'remaining time': '29:27:17', 'throughput': '8560.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:05:16,841 >> {'loss': 0.6771, 'grad_norm': 16.43646240234375, 'learning_rate': 9.905377798788547e-07, 'epoch': 0.11228641171684296, 'num_input_tokens_seen': 868220928, 'completed': '11.23% (414 / 3_687)', 'remaining time': '29:26:17', 'throughput': '9039.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:05:45,040 >> {'loss': 0.4444, 'grad_norm': 14.17488956451416, 'learning_rate': 9.904552539863452e-07, 'epoch': 0.11255763493355031, 'num_input_tokens_seen': 870318080, 'completed': '11.26% (415 / 3_687)', 'remaining time': '29:25:12', 'throughput': '9296.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:06:15,682 >> {'loss': 0.7704, 'grad_norm': 17.454513549804688, 'learning_rate': 9.903723736322075e-07, 'epoch': 0.11282885815025766, 'num_input_tokens_seen': 872415232, 'completed': '11.28% (416 / 3_687)', 'remaining time': '29:24:26', 'throughput': '8554.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:06:46,798 >> {'loss': 0.813, 'grad_norm': 18.859848022460938, 'learning_rate': 9.90289138883141e-07, 'epoch': 0.11310008136696502, 'num_input_tokens_seen': 874512384, 'completed': '11.31% (417 / 3_687)', 'remaining time': '29:23:44', 'throughput': '8424.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:07:12,891 >> {'loss': 0.6002, 'grad_norm': 21.147764205932617, 'learning_rate': 9.9020554980613e-07, 'epoch': 0.11337130458367237, 'num_input_tokens_seen': 876609536, 'completed': '11.34% (418 / 3_687)', 'remaining time': '29:22:23', 'throughput': '10046.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:07:40,605 >> {'loss': 0.9423, 'grad_norm': 20.489925384521484, 'learning_rate': 9.901216064684434e-07, 'epoch': 0.11364252780037971, 'num_input_tokens_seen': 878706688, 'completed': '11.36% (419 / 3_687)', 'remaining time': '29:21:14', 'throughput': '9458.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:08:11,268 >> {'loss': 0.4228, 'grad_norm': 12.808001518249512, 'learning_rate': 9.900373089376357e-07, 'epoch': 0.11391375101708706, 'num_input_tokens_seen': 880803840, 'completed': '11.39% (420 / 3_687)', 'remaining time': '29:20:29', 'throughput': '8549.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:08:42,085 >> {'loss': 0.3785, 'grad_norm': 12.175069808959961, 'learning_rate': 9.899526572815465e-07, 'epoch': 0.11418497423379441, 'num_input_tokens_seen': 882900992, 'completed': '11.42% (421 / 3_687)', 'remaining time': '29:19:45', 'throughput': '8506.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:09:15,592 >> {'loss': 0.4116, 'grad_norm': 12.485739707946777, 'learning_rate': 9.898676515683001e-07, 'epoch': 0.11445619745050176, 'num_input_tokens_seen': 884998144, 'completed': '11.45% (422 / 3_687)', 'remaining time': '29:19:21', 'throughput': '7823.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:09:45,440 >> {'loss': 0.5853, 'grad_norm': 16.668554306030273, 'learning_rate': 9.897822918663062e-07, 'epoch': 0.11472742066720912, 'num_input_tokens_seen': 887095296, 'completed': '11.47% (423 / 3_687)', 'remaining time': '29:18:30', 'throughput': '8782.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:10:17,163 >> {'loss': 0.6277, 'grad_norm': 20.581716537475586, 'learning_rate': 9.896965782442584e-07, 'epoch': 0.11499864388391647, 'num_input_tokens_seen': 889192448, 'completed': '11.50% (424 / 3_687)', 'remaining time': '29:17:53', 'throughput': '8263.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:10:49,543 >> {'loss': 0.5649, 'grad_norm': 17.985069274902344, 'learning_rate': 9.896105107711365e-07, 'epoch': 0.11526986710062381, 'num_input_tokens_seen': 891289600, 'completed': '11.53% (425 / 3_687)', 'remaining time': '29:17:21', 'throughput': '8095.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:11:20,703 >> {'loss': 0.5382, 'grad_norm': 16.7568302154541, 'learning_rate': 9.895240895162037e-07, 'epoch': 0.11554109031733116, 'num_input_tokens_seen': 893386752, 'completed': '11.55% (426 / 3_687)', 'remaining time': '29:16:40', 'throughput': '8412.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:11:52,440 >> {'loss': 0.493, 'grad_norm': 14.392632484436035, 'learning_rate': 9.89437314549009e-07, 'epoch': 0.11581231353403851, 'num_input_tokens_seen': 895483904, 'completed': '11.58% (427 / 3_687)', 'remaining time': '29:16:03', 'throughput': '8259.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:12:25,584 >> {'loss': 0.2819, 'grad_norm': 11.781503677368164, 'learning_rate': 9.89350185939385e-07, 'epoch': 0.11608353675074587, 'num_input_tokens_seen': 897581056, 'completed': '11.61% (428 / 3_687)', 'remaining time': '29:15:37', 'throughput': '7909.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:12:54,902 >> {'loss': 0.8497, 'grad_norm': 18.502288818359375, 'learning_rate': 9.8926270375745e-07, 'epoch': 0.11635475996745322, 'num_input_tokens_seen': 899678208, 'completed': '11.64% (429 / 3_687)', 'remaining time': '29:14:42', 'throughput': '8941.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:13:25,168 >> {'loss': 0.4923, 'grad_norm': 13.56667709350586, 'learning_rate': 9.891748680736064e-07, 'epoch': 0.11662598318416056, 'num_input_tokens_seen': 901775360, 'completed': '11.66% (430 / 3_687)', 'remaining time': '29:13:54', 'throughput': '8661.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:13:57,625 >> {'loss': 0.6438, 'grad_norm': 19.49355125427246, 'learning_rate': 9.890866789585407e-07, 'epoch': 0.11689720640086791, 'num_input_tokens_seen': 903872512, 'completed': '11.69% (431 / 3_687)', 'remaining time': '29:13:23', 'throughput': '8076.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:14:28,784 >> {'loss': 0.3834, 'grad_norm': 11.730159759521484, 'learning_rate': 9.889981364832245e-07, 'epoch': 0.11716842961757526, 'num_input_tokens_seen': 905969664, 'completed': '11.72% (432 / 3_687)', 'remaining time': '29:12:42', 'throughput': '8413.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:14:58,762 >> {'loss': 0.3682, 'grad_norm': 10.967796325683594, 'learning_rate': 9.889092407189129e-07, 'epoch': 0.11743965283428262, 'num_input_tokens_seen': 908066816, 'completed': '11.74% (433 / 3_687)', 'remaining time': '29:11:52', 'throughput': '8744.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:15:29,200 >> {'loss': 0.454, 'grad_norm': 16.377155303955078, 'learning_rate': 9.88819991737146e-07, 'epoch': 0.11771087605098997, 'num_input_tokens_seen': 910163968, 'completed': '11.77% (434 / 3_687)', 'remaining time': '29:11:06', 'throughput': '8612.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:15:59,184 >> {'loss': 0.6401, 'grad_norm': 15.872443199157715, 'learning_rate': 9.887303896097483e-07, 'epoch': 0.11798209926769732, 'num_input_tokens_seen': 912261120, 'completed': '11.80% (435 / 3_687)', 'remaining time': '29:10:16', 'throughput': '8742.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:16:25,658 >> {'loss': 0.7127, 'grad_norm': 15.673075675964355, 'learning_rate': 9.88640434408828e-07, 'epoch': 0.11825332248440466, 'num_input_tokens_seen': 914358272, 'completed': '11.83% (436 / 3_687)', 'remaining time': '29:09:00', 'throughput': '9901.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:16:56,543 >> {'loss': 0.4442, 'grad_norm': 18.29340934753418, 'learning_rate': 9.885501262067776e-07, 'epoch': 0.11852454570111201, 'num_input_tokens_seen': 916455424, 'completed': '11.85% (437 / 3_687)', 'remaining time': '29:08:18', 'throughput': '8487.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:17:26,190 >> {'loss': 1.0107, 'grad_norm': 21.140331268310547, 'learning_rate': 9.884594650762734e-07, 'epoch': 0.11879576891781937, 'num_input_tokens_seen': 918552576, 'completed': '11.88% (438 / 3_687)', 'remaining time': '29:07:26', 'throughput': '8842.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:17:57,645 >> {'loss': 0.4715, 'grad_norm': 21.628915786743164, 'learning_rate': 9.883684510902767e-07, 'epoch': 0.11906699213452672, 'num_input_tokens_seen': 920649728, 'completed': '11.91% (439 / 3_687)', 'remaining time': '29:06:48', 'throughput': '8334.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:18:25,403 >> {'loss': 0.7166, 'grad_norm': 17.046772003173828, 'learning_rate': 9.882770843220316e-07, 'epoch': 0.11933821535123407, 'num_input_tokens_seen': 922746880, 'completed': '11.93% (440 / 3_687)', 'remaining time': '29:05:42', 'throughput': '9443.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:18:56,131 >> {'loss': 0.4021, 'grad_norm': 13.360857963562012, 'learning_rate': 9.881853648450667e-07, 'epoch': 0.11960943856794141, 'num_input_tokens_seen': 924844032, 'completed': '11.96% (441 / 3_687)', 'remaining time': '29:04:59', 'throughput': '8531.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:19:30,048 >> {'loss': 0.3842, 'grad_norm': 12.711376190185547, 'learning_rate': 9.880932927331942e-07, 'epoch': 0.11988066178464876, 'num_input_tokens_seen': 926941184, 'completed': '11.99% (442 / 3_687)', 'remaining time': '29:04:39', 'throughput': '7728.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:20:00,284 >> {'loss': 0.5039, 'grad_norm': 13.425957679748535, 'learning_rate': 9.880008680605104e-07, 'epoch': 0.12015188500135611, 'num_input_tokens_seen': 929038336, 'completed': '12.02% (443 / 3_687)', 'remaining time': '29:03:52', 'throughput': '8669.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:20:30,274 >> {'loss': 0.3667, 'grad_norm': 14.570152282714844, 'learning_rate': 9.879080909013955e-07, 'epoch': 0.12042310821806347, 'num_input_tokens_seen': 931135488, 'completed': '12.04% (444 / 3_687)', 'remaining time': '29:03:03', 'throughput': '8741.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:21:01,450 >> {'loss': 0.6197, 'grad_norm': 15.635329246520996, 'learning_rate': 9.878149613305125e-07, 'epoch': 0.12069433143477082, 'num_input_tokens_seen': 933232640, 'completed': '12.07% (445 / 3_687)', 'remaining time': '29:02:23', 'throughput': '8408.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:21:32,484 >> {'loss': 0.3233, 'grad_norm': 10.609899520874023, 'learning_rate': 9.877214794228087e-07, 'epoch': 0.12096555465147817, 'num_input_tokens_seen': 935329792, 'completed': '12.10% (446 / 3_687)', 'remaining time': '29:01:42', 'throughput': '8446.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:22:03,774 >> {'loss': 0.664, 'grad_norm': 18.10693359375, 'learning_rate': 9.876276452535147e-07, 'epoch': 0.12123677786818551, 'num_input_tokens_seen': 937426944, 'completed': '12.12% (447 / 3_687)', 'remaining time': '29:01:02', 'throughput': '8377.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:22:33,869 >> {'loss': 0.3728, 'grad_norm': 12.761764526367188, 'learning_rate': 9.87533458898145e-07, 'epoch': 0.12150800108489286, 'num_input_tokens_seen': 939524096, 'completed': '12.15% (448 / 3_687)', 'remaining time': '29:00:15', 'throughput': '8710.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:23:04,901 >> {'loss': 0.4342, 'grad_norm': 11.233665466308594, 'learning_rate': 9.874389204324967e-07, 'epoch': 0.12177922430160022, 'num_input_tokens_seen': 941621248, 'completed': '12.18% (449 / 3_687)', 'remaining time': '28:59:34', 'throughput': '8447.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:23:35,531 >> {'loss': 0.5682, 'grad_norm': 14.0928955078125, 'learning_rate': 9.873440299326513e-07, 'epoch': 0.12205044751830757, 'num_input_tokens_seen': 943718400, 'completed': '12.21% (450 / 3_687)', 'remaining time': '28:58:50', 'throughput': '8558.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:24:08,845 >> {'loss': 0.5666, 'grad_norm': 14.01526927947998, 'learning_rate': 9.872487874749726e-07, 'epoch': 0.12232167073501492, 'num_input_tokens_seen': 945815552, 'completed': '12.23% (451 / 3_687)', 'remaining time': '28:58:26', 'throughput': '7868.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:24:40,405 >> {'loss': 0.4609, 'grad_norm': 15.093655586242676, 'learning_rate': 9.871531931361084e-07, 'epoch': 0.12259289395172226, 'num_input_tokens_seen': 947912704, 'completed': '12.26% (452 / 3_687)', 'remaining time': '28:57:49', 'throughput': '8306.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:25:12,918 >> {'loss': 0.4889, 'grad_norm': 16.862146377563477, 'learning_rate': 9.870572469929892e-07, 'epoch': 0.12286411716842961, 'num_input_tokens_seen': 950009856, 'completed': '12.29% (453 / 3_687)', 'remaining time': '28:57:18', 'throughput': '8062.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:25:42,786 >> {'loss': 0.4386, 'grad_norm': 14.488612174987793, 'learning_rate': 9.869609491228288e-07, 'epoch': 0.12313534038513697, 'num_input_tokens_seen': 952107008, 'completed': '12.31% (454 / 3_687)', 'remaining time': '28:56:29', 'throughput': '8776.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:26:12,986 >> {'loss': 0.3064, 'grad_norm': 10.171273231506348, 'learning_rate': 9.868642996031243e-07, 'epoch': 0.12340656360184432, 'num_input_tokens_seen': 954204160, 'completed': '12.34% (455 / 3_687)', 'remaining time': '28:55:43', 'throughput': '8680.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:26:42,802 >> {'loss': 0.4111, 'grad_norm': 13.183239936828613, 'learning_rate': 9.867672985116553e-07, 'epoch': 0.12367778681855167, 'num_input_tokens_seen': 956301312, 'completed': '12.37% (456 / 3_687)', 'remaining time': '28:54:53', 'throughput': '8792.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:27:14,692 >> {'loss': 0.5021, 'grad_norm': 13.565240859985352, 'learning_rate': 9.866699459264846e-07, 'epoch': 0.12394901003525902, 'num_input_tokens_seen': 958398464, 'completed': '12.39% (457 / 3_687)', 'remaining time': '28:54:19', 'throughput': '8220.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:27:46,920 >> {'loss': 0.4652, 'grad_norm': 14.678339958190918, 'learning_rate': 9.865722419259582e-07, 'epoch': 0.12422023325196636, 'num_input_tokens_seen': 960495616, 'completed': '12.42% (458 / 3_687)', 'remaining time': '28:53:47', 'throughput': '8134.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:28:15,352 >> {'loss': 0.91, 'grad_norm': 20.48042869567871, 'learning_rate': 9.864741865887042e-07, 'epoch': 0.12449145646867373, 'num_input_tokens_seen': 962592768, 'completed': '12.45% (459 / 3_687)', 'remaining time': '28:52:48', 'throughput': '9219.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:28:46,501 >> {'loss': 0.6582, 'grad_norm': 20.1795711517334, 'learning_rate': 9.86375779993634e-07, 'epoch': 0.12476267968538107, 'num_input_tokens_seen': 964689920, 'completed': '12.48% (460 / 3_687)', 'remaining time': '28:52:08', 'throughput': '8415.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:29:17,084 >> {'loss': 0.6452, 'grad_norm': 28.812326431274414, 'learning_rate': 9.86277022219941e-07, 'epoch': 0.1250339029020884, 'num_input_tokens_seen': 966787072, 'completed': '12.50% (461 / 3_687)', 'remaining time': '28:51:25', 'throughput': '8571.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:29:44,092 >> {'loss': 0.9777, 'grad_norm': 21.05647850036621, 'learning_rate': 9.861779133471025e-07, 'epoch': 0.12530512611879577, 'num_input_tokens_seen': 968884224, 'completed': '12.53% (462 / 3_687)', 'remaining time': '28:50:16', 'throughput': '9706.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:30:14,199 >> {'loss': 0.3998, 'grad_norm': 18.095561981201172, 'learning_rate': 9.86078453454877e-07, 'epoch': 0.12557634933550313, 'num_input_tokens_seen': 970981376, 'completed': '12.56% (463 / 3_687)', 'remaining time': '28:49:29', 'throughput': '8707.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:30:45,753 >> {'loss': 0.4588, 'grad_norm': 12.681943893432617, 'learning_rate': 9.859786426233061e-07, 'epoch': 0.12584757255221046, 'num_input_tokens_seen': 973078528, 'completed': '12.58% (464 / 3_687)', 'remaining time': '28:48:53', 'throughput': '8308.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:31:17,410 >> {'loss': 0.3087, 'grad_norm': 11.929615020751953, 'learning_rate': 9.85878480932714e-07, 'epoch': 0.12611879576891782, 'num_input_tokens_seen': 975175680, 'completed': '12.61% (465 / 3_687)', 'remaining time': '28:48:17', 'throughput': '8280.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:31:45,966 >> {'loss': 0.8953, 'grad_norm': 20.00129508972168, 'learning_rate': 9.857779684637068e-07, 'epoch': 0.12639001898562516, 'num_input_tokens_seen': 977272832, 'completed': '12.64% (466 / 3_687)', 'remaining time': '28:47:20', 'throughput': '9180.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:32:19,304 >> {'loss': 0.3192, 'grad_norm': 12.261861801147461, 'learning_rate': 9.856771052971733e-07, 'epoch': 0.12666124220233252, 'num_input_tokens_seen': 979369984, 'completed': '12.67% (467 / 3_687)', 'remaining time': '28:46:56', 'throughput': '7863.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:32:53,444 >> {'loss': 0.5931, 'grad_norm': 15.92088794708252, 'learning_rate': 9.85575891514284e-07, 'epoch': 0.12693246541903988, 'num_input_tokens_seen': 981467136, 'completed': '12.69% (468 / 3_687)', 'remaining time': '28:46:37', 'throughput': '7678.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:33:25,394 >> {'loss': 0.3394, 'grad_norm': 12.561171531677246, 'learning_rate': 9.85474327196492e-07, 'epoch': 0.12720368863574721, 'num_input_tokens_seen': 983564288, 'completed': '12.72% (469 / 3_687)', 'remaining time': '28:46:03', 'throughput': '8204.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:33:58,148 >> {'loss': 0.4335, 'grad_norm': 16.519527435302734, 'learning_rate': 9.853724124255328e-07, 'epoch': 0.12747491185245458, 'num_input_tokens_seen': 985661440, 'completed': '12.75% (470 / 3_687)', 'remaining time': '28:45:35', 'throughput': '8003.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:34:30,290 >> {'loss': 0.6265, 'grad_norm': 23.729719161987305, 'learning_rate': 9.85270147283423e-07, 'epoch': 0.1277461350691619, 'num_input_tokens_seen': 987758592, 'completed': '12.77% (471 / 3_687)', 'remaining time': '28:45:02', 'throughput': '8155.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:35:00,764 >> {'loss': 0.8864, 'grad_norm': 20.1112003326416, 'learning_rate': 9.85167531852462e-07, 'epoch': 0.12801735828586927, 'num_input_tokens_seen': 989855744, 'completed': '12.80% (472 / 3_687)', 'remaining time': '28:44:19', 'throughput': '8602.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:35:33,486 >> {'loss': 0.3044, 'grad_norm': 11.423717498779297, 'learning_rate': 9.850645662152308e-07, 'epoch': 0.12828858150257663, 'num_input_tokens_seen': 991952896, 'completed': '12.83% (473 / 3_687)', 'remaining time': '28:43:50', 'throughput': '8011.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:36:04,049 >> {'loss': 0.4784, 'grad_norm': 15.77081298828125, 'learning_rate': 9.84961250454592e-07, 'epoch': 0.12855980471928397, 'num_input_tokens_seen': 994050048, 'completed': '12.86% (474 / 3_687)', 'remaining time': '28:43:07', 'throughput': '8577.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:36:35,695 >> {'loss': 0.4869, 'grad_norm': 14.118724822998047, 'learning_rate': 9.848575846536902e-07, 'epoch': 0.12883102793599133, 'num_input_tokens_seen': 996147200, 'completed': '12.88% (475 / 3_687)', 'remaining time': '28:42:31', 'throughput': '8283.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:37:05,908 >> {'loss': 0.5372, 'grad_norm': 14.419587135314941, 'learning_rate': 9.847535688959523e-07, 'epoch': 0.12910225115269866, 'num_input_tokens_seen': 998244352, 'completed': '12.91% (476 / 3_687)', 'remaining time': '28:41:46', 'throughput': '8676.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:37:38,691 >> {'loss': 0.5932, 'grad_norm': 17.4903507232666, 'learning_rate': 9.846492032650855e-07, 'epoch': 0.12937347436940602, 'num_input_tokens_seen': 1000341504, 'completed': '12.94% (477 / 3_687)', 'remaining time': '28:41:18', 'throughput': '7996.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:38:10,180 >> {'loss': 0.4332, 'grad_norm': 13.698707580566406, 'learning_rate': 9.845444878450794e-07, 'epoch': 0.12964469758611338, 'num_input_tokens_seen': 1002438656, 'completed': '12.96% (478 / 3_687)', 'remaining time': '28:40:41', 'throughput': '8325.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:38:39,822 >> {'loss': 0.5934, 'grad_norm': 20.30847930908203, 'learning_rate': 9.844394227202053e-07, 'epoch': 0.12991592080282072, 'num_input_tokens_seen': 1004535808, 'completed': '12.99% (479 / 3_687)', 'remaining time': '28:39:52', 'throughput': '8843.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:39:08,751 >> {'loss': 0.4459, 'grad_norm': 14.15876293182373, 'learning_rate': 9.843340079750154e-07, 'epoch': 0.13018714401952808, 'num_input_tokens_seen': 1006632960, 'completed': '13.02% (480 / 3_687)', 'remaining time': '28:38:58', 'throughput': '9061.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:39:41,364 >> {'loss': 0.3993, 'grad_norm': 12.466696739196777, 'learning_rate': 9.842282436943435e-07, 'epoch': 0.1304583672362354, 'num_input_tokens_seen': 1008730112, 'completed': '13.05% (481 / 3_687)', 'remaining time': '28:38:29', 'throughput': '8037.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:40:12,386 >> {'loss': 0.517, 'grad_norm': 17.355510711669922, 'learning_rate': 9.841221299633049e-07, 'epoch': 0.13072959045294277, 'num_input_tokens_seen': 1010827264, 'completed': '13.07% (482 / 3_687)', 'remaining time': '28:37:49', 'throughput': '8450.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:40:44,615 >> {'loss': 0.7, 'grad_norm': 23.515899658203125, 'learning_rate': 9.840156668672953e-07, 'epoch': 0.13100081366965013, 'num_input_tokens_seen': 1012924416, 'completed': '13.10% (483 / 3_687)', 'remaining time': '28:37:17', 'throughput': '8133.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:41:15,872 >> {'loss': 0.3053, 'grad_norm': 8.689852714538574, 'learning_rate': 9.839088544919927e-07, 'epoch': 0.13127203688635747, 'num_input_tokens_seen': 1015021568, 'completed': '13.13% (484 / 3_687)', 'remaining time': '28:36:39', 'throughput': '8386.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:41:49,052 >> {'loss': 0.4763, 'grad_norm': 16.73533058166504, 'learning_rate': 9.838016929233555e-07, 'epoch': 0.13154326010306483, 'num_input_tokens_seen': 1017118720, 'completed': '13.15% (485 / 3_687)', 'remaining time': '28:36:14', 'throughput': '7900.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:42:22,636 >> {'loss': 0.3931, 'grad_norm': 16.722810745239258, 'learning_rate': 9.836941822476232e-07, 'epoch': 0.13181448331977216, 'num_input_tokens_seen': 1019215872, 'completed': '13.18% (486 / 3_687)', 'remaining time': '28:35:51', 'throughput': '7805.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:42:53,511 >> {'loss': 0.63, 'grad_norm': 16.654083251953125, 'learning_rate': 9.835863225513163e-07, 'epoch': 0.13208570653647952, 'num_input_tokens_seen': 1021313024, 'completed': '13.21% (487 / 3_687)', 'remaining time': '28:35:10', 'throughput': '8490.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:43:21,315 >> {'loss': 0.8202, 'grad_norm': 17.620363235473633, 'learning_rate': 9.83478113921236e-07, 'epoch': 0.13235692975318689, 'num_input_tokens_seen': 1023410176, 'completed': '13.24% (488 / 3_687)', 'remaining time': '28:34:10', 'throughput': '9428.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:43:50,698 >> {'loss': 0.4924, 'grad_norm': 14.268228530883789, 'learning_rate': 9.833695564444652e-07, 'epoch': 0.13262815296989422, 'num_input_tokens_seen': 1025507328, 'completed': '13.26% (489 / 3_687)', 'remaining time': '28:33:19', 'throughput': '8921.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:44:21,493 >> {'loss': 0.4732, 'grad_norm': 12.854048728942871, 'learning_rate': 9.832606502083658e-07, 'epoch': 0.13289937618660158, 'num_input_tokens_seen': 1027604480, 'completed': '13.29% (490 / 3_687)', 'remaining time': '28:32:39', 'throughput': '8512.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:44:52,531 >> {'loss': 0.5987, 'grad_norm': 17.436908721923828, 'learning_rate': 9.83151395300582e-07, 'epoch': 0.13317059940330891, 'num_input_tokens_seen': 1029701632, 'completed': '13.32% (491 / 3_687)', 'remaining time': '28:31:59', 'throughput': '8446.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:45:24,239 >> {'loss': 0.4847, 'grad_norm': 13.574828147888184, 'learning_rate': 9.83041791809038e-07, 'epoch': 0.13344182262001628, 'num_input_tokens_seen': 1031798784, 'completed': '13.34% (492 / 3_687)', 'remaining time': '28:31:24', 'throughput': '8267.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:45:58,065 >> {'loss': 0.5305, 'grad_norm': 18.534931182861328, 'learning_rate': 9.829318398219385e-07, 'epoch': 0.13371304583672364, 'num_input_tokens_seen': 1033895936, 'completed': '13.37% (493 / 3_687)', 'remaining time': '28:31:03', 'throughput': '7749.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:46:29,036 >> {'loss': 0.3694, 'grad_norm': 13.970446586608887, 'learning_rate': 9.828215394277686e-07, 'epoch': 0.13398426905343097, 'num_input_tokens_seen': 1035993088, 'completed': '13.40% (494 / 3_687)', 'remaining time': '28:30:23', 'throughput': '8464.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:46:58,865 >> {'loss': 0.5241, 'grad_norm': 15.922163009643555, 'learning_rate': 9.827108907152937e-07, 'epoch': 0.13425549227013833, 'num_input_tokens_seen': 1038090240, 'completed': '13.43% (495 / 3_687)', 'remaining time': '28:29:36', 'throughput': '8788.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:47:29,438 >> {'loss': 0.6606, 'grad_norm': 15.5534029006958, 'learning_rate': 9.825998937735599e-07, 'epoch': 0.13452671548684567, 'num_input_tokens_seen': 1040187392, 'completed': '13.45% (496 / 3_687)', 'remaining time': '28:28:54', 'throughput': '8574.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:47:59,640 >> {'loss': 0.7119, 'grad_norm': 18.83726692199707, 'learning_rate': 9.824885486918932e-07, 'epoch': 0.13479793870355303, 'num_input_tokens_seen': 1042284544, 'completed': '13.48% (497 / 3_687)', 'remaining time': '28:28:10', 'throughput': '8679.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:48:28,894 >> {'loss': 0.8065, 'grad_norm': 19.1335506439209, 'learning_rate': 9.823768555599e-07, 'epoch': 0.13506916192026036, 'num_input_tokens_seen': 1044381696, 'completed': '13.51% (498 / 3_687)', 'remaining time': '28:27:19', 'throughput': '8960.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:48:59,124 >> {'loss': 0.324, 'grad_norm': 13.459641456604004, 'learning_rate': 9.822648144674664e-07, 'epoch': 0.13534038513696772, 'num_input_tokens_seen': 1046478848, 'completed': '13.53% (499 / 3_687)', 'remaining time': '28:26:35', 'throughput': '8671.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:49:30,281 >> {'loss': 0.3856, 'grad_norm': 11.397218704223633, 'learning_rate': 9.821524255047592e-07, 'epoch': 0.13561160835367508, 'num_input_tokens_seen': 1048576000, 'completed': '13.56% (500 / 3_687)', 'remaining time': '28:25:57', 'throughput': '8413.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:50:01,086 >> {'loss': 0.4602, 'grad_norm': 13.145438194274902, 'learning_rate': 9.820396887622245e-07, 'epoch': 0.13588283157038242, 'num_input_tokens_seen': 1050673152, 'completed': '13.59% (501 / 3_687)', 'remaining time': '28:25:16', 'throughput': '8509.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:50:32,933 >> {'loss': 0.3918, 'grad_norm': 12.989845275878906, 'learning_rate': 9.819266043305887e-07, 'epoch': 0.13615405478708978, 'num_input_tokens_seen': 1052770304, 'completed': '13.62% (502 / 3_687)', 'remaining time': '28:24:42', 'throughput': '8231.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:51:03,805 >> {'loss': 0.5483, 'grad_norm': 18.566017150878906, 'learning_rate': 9.818131723008576e-07, 'epoch': 0.1364252780037971, 'num_input_tokens_seen': 1054867456, 'completed': '13.64% (503 / 3_687)', 'remaining time': '28:24:02', 'throughput': '8491.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:51:37,363 >> {'loss': 0.8955, 'grad_norm': 20.60564613342285, 'learning_rate': 9.816993927643174e-07, 'epoch': 0.13669650122050447, 'num_input_tokens_seen': 1056964608, 'completed': '13.67% (504 / 3_687)', 'remaining time': '28:23:39', 'throughput': '7811.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:52:10,834 >> {'loss': 0.4461, 'grad_norm': 13.428114891052246, 'learning_rate': 9.815852658125332e-07, 'epoch': 0.13696772443721184, 'num_input_tokens_seen': 1059061760, 'completed': '13.70% (505 / 3_687)', 'remaining time': '28:23:16', 'throughput': '7832.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:52:41,641 >> {'loss': 0.6438, 'grad_norm': 15.920856475830078, 'learning_rate': 9.8147079153735e-07, 'epoch': 0.13723894765391917, 'num_input_tokens_seen': 1061158912, 'completed': '13.72% (506 / 3_687)', 'remaining time': '28:22:35', 'throughput': '8509.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:53:13,706 >> {'loss': 0.3101, 'grad_norm': 11.576249122619629, 'learning_rate': 9.813559700308925e-07, 'epoch': 0.13751017087062653, 'num_input_tokens_seen': 1063256064, 'completed': '13.75% (507 / 3_687)', 'remaining time': '28:22:03', 'throughput': '8175.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:53:43,825 >> {'loss': 0.7386, 'grad_norm': 20.885211944580078, 'learning_rate': 9.812408013855646e-07, 'epoch': 0.13778139408733386, 'num_input_tokens_seen': 1065353216, 'completed': '13.78% (508 / 3_687)', 'remaining time': '28:21:18', 'throughput': '8703.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:54:12,316 >> {'loss': 0.6981, 'grad_norm': 16.579875946044922, 'learning_rate': 9.811252856940496e-07, 'epoch': 0.13805261730404123, 'num_input_tokens_seen': 1067450368, 'completed': '13.81% (509 / 3_687)', 'remaining time': '28:20:24', 'throughput': '9200.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:54:44,536 >> {'loss': 0.2932, 'grad_norm': 10.057287216186523, 'learning_rate': 9.810094230493104e-07, 'epoch': 0.1383238405207486, 'num_input_tokens_seen': 1069547520, 'completed': '13.83% (510 / 3_687)', 'remaining time': '28:19:52', 'throughput': '8136.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:55:12,762 >> {'loss': 0.5647, 'grad_norm': 17.37000846862793, 'learning_rate': 9.808932135445885e-07, 'epoch': 0.13859506373745592, 'num_input_tokens_seen': 1071644672, 'completed': '13.86% (511 / 3_687)', 'remaining time': '28:18:56', 'throughput': '9287.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:55:42,855 >> {'loss': 0.7026, 'grad_norm': 15.917064666748047, 'learning_rate': 9.807766572734052e-07, 'epoch': 0.13886628695416328, 'num_input_tokens_seen': 1073741824, 'completed': '13.89% (512 / 3_687)', 'remaining time': '28:18:12', 'throughput': '8710.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:56:11,893 >> {'loss': 0.416, 'grad_norm': 11.753175735473633, 'learning_rate': 9.806597543295603e-07, 'epoch': 0.13913751017087062, 'num_input_tokens_seen': 1075838976, 'completed': '13.91% (513 / 3_687)', 'remaining time': '28:17:21', 'throughput': '9027.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:56:41,340 >> {'loss': 0.5972, 'grad_norm': 18.979516983032227, 'learning_rate': 9.80542504807133e-07, 'epoch': 0.13940873338757798, 'num_input_tokens_seen': 1077936128, 'completed': '13.94% (514 / 3_687)', 'remaining time': '28:16:32', 'throughput': '8902.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:57:13,023 >> {'loss': 0.4001, 'grad_norm': 28.44046401977539, 'learning_rate': 9.804249088004812e-07, 'epoch': 0.13967995660428534, 'num_input_tokens_seen': 1080033280, 'completed': '13.97% (515 / 3_687)', 'remaining time': '28:15:58', 'throughput': '8273.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:57:46,978 >> {'loss': 0.3327, 'grad_norm': 12.617019653320312, 'learning_rate': 9.803069664042416e-07, 'epoch': 0.13995117982099267, 'num_input_tokens_seen': 1082130432, 'completed': '14.00% (516 / 3_687)', 'remaining time': '28:15:37', 'throughput': '7720.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:58:19,236 >> {'loss': 0.5243, 'grad_norm': 14.718525886535645, 'learning_rate': 9.801886777133297e-07, 'epoch': 0.14022240303770003, 'num_input_tokens_seen': 1084227584, 'completed': '14.02% (517 / 3_687)', 'remaining time': '28:15:06', 'throughput': '8126.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:58:49,936 >> {'loss': 0.5279, 'grad_norm': 17.5297794342041, 'learning_rate': 9.8007004282294e-07, 'epoch': 0.14049362625440737, 'num_input_tokens_seen': 1086324736, 'completed': '14.05% (518 / 3_687)', 'remaining time': '28:14:26', 'throughput': '8538.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:59:21,811 >> {'loss': 0.4663, 'grad_norm': 13.261266708374023, 'learning_rate': 9.799510618285454e-07, 'epoch': 0.14076484947111473, 'num_input_tokens_seen': 1088421888, 'completed': '14.08% (519 / 3_687)', 'remaining time': '28:13:52', 'throughput': '8224.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 15:59:54,295 >> {'loss': 0.7519, 'grad_norm': 22.933935165405273, 'learning_rate': 9.79831734825897e-07, 'epoch': 0.1410360726878221, 'num_input_tokens_seen': 1090519040, 'completed': '14.10% (520 / 3_687)', 'remaining time': '28:13:23', 'throughput': '8069.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:00:26,425 >> {'loss': 0.7251, 'grad_norm': 20.257604598999023, 'learning_rate': 9.797120619110245e-07, 'epoch': 0.14130729590452942, 'num_input_tokens_seen': 1092616192, 'completed': '14.13% (521 / 3_687)', 'remaining time': '28:12:51', 'throughput': '8158.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:00:55,882 >> {'loss': 0.7534, 'grad_norm': 19.605735778808594, 'learning_rate': 9.795920431802365e-07, 'epoch': 0.14157851912123678, 'num_input_tokens_seen': 1094713344, 'completed': '14.16% (522 / 3_687)', 'remaining time': '28:12:03', 'throughput': '8899.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:01:27,320 >> {'loss': 0.5247, 'grad_norm': 13.443857192993164, 'learning_rate': 9.794716787301194e-07, 'epoch': 0.14184974233794412, 'num_input_tokens_seen': 1096810496, 'completed': '14.18% (523 / 3_687)', 'remaining time': '28:11:27', 'throughput': '8338.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:01:57,133 >> {'loss': 0.7223, 'grad_norm': 21.489839553833008, 'learning_rate': 9.793509686575378e-07, 'epoch': 0.14212096555465148, 'num_input_tokens_seen': 1098907648, 'completed': '14.21% (524 / 3_687)', 'remaining time': '28:10:41', 'throughput': '8792.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:02:27,302 >> {'loss': 0.6552, 'grad_norm': 15.88458251953125, 'learning_rate': 9.792299130596346e-07, 'epoch': 0.14239218877135884, 'num_input_tokens_seen': 1101004800, 'completed': '14.24% (525 / 3_687)', 'remaining time': '28:09:58', 'throughput': '8689.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:03:00,636 >> {'loss': 0.8404, 'grad_norm': 20.70093536376953, 'learning_rate': 9.79108512033831e-07, 'epoch': 0.14266341198806617, 'num_input_tokens_seen': 1103101952, 'completed': '14.27% (526 / 3_687)', 'remaining time': '28:09:33', 'throughput': '7863.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:03:31,248 >> {'loss': 0.6306, 'grad_norm': 17.892988204956055, 'learning_rate': 9.789867656778254e-07, 'epoch': 0.14293463520477354, 'num_input_tokens_seen': 1105199104, 'completed': '14.29% (527 / 3_687)', 'remaining time': '28:08:52', 'throughput': '8563.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:04:01,432 >> {'loss': 0.6665, 'grad_norm': 16.286060333251953, 'learning_rate': 9.788646740895952e-07, 'epoch': 0.14320585842148087, 'num_input_tokens_seen': 1107296256, 'completed': '14.32% (528 / 3_687)', 'remaining time': '28:08:09', 'throughput': '8684.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:04:31,486 >> {'loss': 0.5283, 'grad_norm': 14.654132843017578, 'learning_rate': 9.787422373673945e-07, 'epoch': 0.14347708163818823, 'num_input_tokens_seen': 1109393408, 'completed': '14.35% (529 / 3_687)', 'remaining time': '28:07:25', 'throughput': '8722.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:05:04,254 >> {'loss': 0.6534, 'grad_norm': 16.374067306518555, 'learning_rate': 9.786194556097564e-07, 'epoch': 0.1437483048548956, 'num_input_tokens_seen': 1111490560, 'completed': '14.37% (530 / 3_687)', 'remaining time': '28:06:57', 'throughput': '8000.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:05:31,471 >> {'loss': 0.928, 'grad_norm': 19.840892791748047, 'learning_rate': 9.784963289154902e-07, 'epoch': 0.14401952807160293, 'num_input_tokens_seen': 1113587712, 'completed': '14.40% (531 / 3_687)', 'remaining time': '28:05:56', 'throughput': '9631.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:06:01,587 >> {'loss': 0.4473, 'grad_norm': 12.525459289550781, 'learning_rate': 9.783728573836843e-07, 'epoch': 0.1442907512883103, 'num_input_tokens_seen': 1115684864, 'completed': '14.43% (532 / 3_687)', 'remaining time': '28:05:13', 'throughput': '8704.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:06:34,229 >> {'loss': 0.387, 'grad_norm': 15.423880577087402, 'learning_rate': 9.782490411137035e-07, 'epoch': 0.14456197450501762, 'num_input_tokens_seen': 1117782016, 'completed': '14.46% (533 / 3_687)', 'remaining time': '28:04:44', 'throughput': '8030.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:07:07,642 >> {'loss': 0.5885, 'grad_norm': 14.608306884765625, 'learning_rate': 9.781248802051904e-07, 'epoch': 0.14483319772172498, 'num_input_tokens_seen': 1119879168, 'completed': '14.48% (534 / 3_687)', 'remaining time': '28:04:20', 'throughput': '7845.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:07:37,963 >> {'loss': 0.674, 'grad_norm': 17.79987144470215, 'learning_rate': 9.780003747580651e-07, 'epoch': 0.14510442093843234, 'num_input_tokens_seen': 1121976320, 'completed': '14.51% (535 / 3_687)', 'remaining time': '28:03:38', 'throughput': '8645.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:08:08,234 >> {'loss': 0.4104, 'grad_norm': 11.311891555786133, 'learning_rate': 9.778755248725248e-07, 'epoch': 0.14537564415513968, 'num_input_tokens_seen': 1124073472, 'completed': '14.54% (536 / 3_687)', 'remaining time': '28:02:55', 'throughput': '8660.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:08:42,371 >> {'loss': 0.5469, 'grad_norm': 13.756656646728516, 'learning_rate': 9.77750330649044e-07, 'epoch': 0.14564686737184704, 'num_input_tokens_seen': 1126170624, 'completed': '14.56% (537 / 3_687)', 'remaining time': '28:02:36', 'throughput': '7679.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:09:10,042 >> {'loss': 0.9223, 'grad_norm': 19.89360809326172, 'learning_rate': 9.776247921883743e-07, 'epoch': 0.14591809058855437, 'num_input_tokens_seen': 1128267776, 'completed': '14.59% (538 / 3_687)', 'remaining time': '28:01:38', 'throughput': '9473.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:09:41,509 >> {'loss': 0.491, 'grad_norm': 14.971158027648926, 'learning_rate': 9.774989095915442e-07, 'epoch': 0.14618931380526173, 'num_input_tokens_seen': 1130364928, 'completed': '14.62% (539 / 3_687)', 'remaining time': '28:01:03', 'throughput': '8330.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:10:09,888 >> {'loss': 0.6727, 'grad_norm': 15.824216842651367, 'learning_rate': 9.77372682959859e-07, 'epoch': 0.14646053702196907, 'num_input_tokens_seen': 1132462080, 'completed': '14.65% (540 / 3_687)', 'remaining time': '28:00:09', 'throughput': '9237.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:10:41,042 >> {'loss': 0.7117, 'grad_norm': 16.35426902770996, 'learning_rate': 9.772461123949015e-07, 'epoch': 0.14673176023867643, 'num_input_tokens_seen': 1134559232, 'completed': '14.67% (541 / 3_687)', 'remaining time': '27:59:32', 'throughput': '8414.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:11:11,807 >> {'loss': 0.772, 'grad_norm': 17.143768310546875, 'learning_rate': 9.771191979985303e-07, 'epoch': 0.1470029834553838, 'num_input_tokens_seen': 1136656384, 'completed': '14.70% (542 / 3_687)', 'remaining time': '27:58:53', 'throughput': '8520.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:11:41,657 >> {'loss': 0.3995, 'grad_norm': 11.403589248657227, 'learning_rate': 9.76991939872882e-07, 'epoch': 0.14727420667209112, 'num_input_tokens_seen': 1138753536, 'completed': '14.73% (543 / 3_687)', 'remaining time': '27:58:08', 'throughput': '8782.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:12:09,203 >> {'loss': 0.5924, 'grad_norm': 13.465624809265137, 'learning_rate': 9.768643381203686e-07, 'epoch': 0.14754542988879848, 'num_input_tokens_seen': 1140850688, 'completed': '14.75% (544 / 3_687)', 'remaining time': '27:57:10', 'throughput': '9516.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:12:37,976 >> {'loss': 0.5763, 'grad_norm': 15.813559532165527, 'learning_rate': 9.767363928436793e-07, 'epoch': 0.14781665310550582, 'num_input_tokens_seen': 1142947840, 'completed': '14.78% (545 / 3_687)', 'remaining time': '27:56:19', 'throughput': '9110.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:13:07,136 >> {'loss': 0.6967, 'grad_norm': 17.59358787536621, 'learning_rate': 9.766081041457795e-07, 'epoch': 0.14808787632221318, 'num_input_tokens_seen': 1145044992, 'completed': '14.81% (546 / 3_687)', 'remaining time': '27:55:31', 'throughput': '8989.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:13:40,355 >> {'loss': 0.7236, 'grad_norm': 15.673956871032715, 'learning_rate': 9.764794721299113e-07, 'epoch': 0.14835909953892054, 'num_input_tokens_seen': 1147142144, 'completed': '14.84% (547 / 3_687)', 'remaining time': '27:55:06', 'throughput': '7891.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:14:16,598 >> {'loss': 0.4046, 'grad_norm': 13.424758911132812, 'learning_rate': 9.763504968995927e-07, 'epoch': 0.14863032275562787, 'num_input_tokens_seen': 1149239296, 'completed': '14.86% (548 / 3_687)', 'remaining time': '27:54:58', 'throughput': '7232.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:14:44,970 >> {'loss': 0.781, 'grad_norm': 18.995798110961914, 'learning_rate': 9.762211785586178e-07, 'epoch': 0.14890154597233524, 'num_input_tokens_seen': 1151336448, 'completed': '14.89% (549 / 3_687)', 'remaining time': '27:54:05', 'throughput': '9239.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:15:14,988 >> {'loss': 0.3885, 'grad_norm': 12.103052139282227, 'learning_rate': 9.760915172110574e-07, 'epoch': 0.14917276918904257, 'num_input_tokens_seen': 1153433600, 'completed': '14.92% (550 / 3_687)', 'remaining time': '27:53:22', 'throughput': '8733.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:15:48,419 >> {'loss': 0.3381, 'grad_norm': 12.997087478637695, 'learning_rate': 9.759615129612579e-07, 'epoch': 0.14944399240574993, 'num_input_tokens_seen': 1155530752, 'completed': '14.94% (551 / 3_687)', 'remaining time': '27:52:58', 'throughput': '7841.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:16:21,404 >> {'loss': 0.5939, 'grad_norm': 15.175359725952148, 'learning_rate': 9.75831165913842e-07, 'epoch': 0.1497152156224573, 'num_input_tokens_seen': 1157627904, 'completed': '14.97% (552 / 3_687)', 'remaining time': '27:52:32', 'throughput': '7947.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:16:50,818 >> {'loss': 0.5293, 'grad_norm': 15.981290817260742, 'learning_rate': 9.757004761737077e-07, 'epoch': 0.14998643883916463, 'num_input_tokens_seen': 1159725056, 'completed': '15.00% (553 / 3_687)', 'remaining time': '27:51:45', 'throughput': '8911.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:17:22,047 >> {'loss': 0.4189, 'grad_norm': 11.93864631652832, 'learning_rate': 9.755694438460293e-07, 'epoch': 0.150257662055872, 'num_input_tokens_seen': 1161822208, 'completed': '15.03% (554 / 3_687)', 'remaining time': '27:51:08', 'throughput': '8394.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:17:52,833 >> {'loss': 0.4993, 'grad_norm': 15.042703628540039, 'learning_rate': 9.754380690362565e-07, 'epoch': 0.15052888527257932, 'num_input_tokens_seen': 1163919360, 'completed': '15.05% (555 / 3_687)', 'remaining time': '27:50:30', 'throughput': '8514.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:18:24,205 >> {'loss': 0.472, 'grad_norm': 14.069710731506348, 'learning_rate': 9.75306351850115e-07, 'epoch': 0.15080010848928668, 'num_input_tokens_seen': 1166016512, 'completed': '15.08% (556 / 3_687)', 'remaining time': '27:49:54', 'throughput': '8356.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:18:53,179 >> {'loss': 0.4091, 'grad_norm': 15.970101356506348, 'learning_rate': 9.751742923936055e-07, 'epoch': 0.15107133170599404, 'num_input_tokens_seen': 1168113664, 'completed': '15.11% (557 / 3_687)', 'remaining time': '27:49:05', 'throughput': '9047.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:19:24,798 >> {'loss': 0.5406, 'grad_norm': 16.250965118408203, 'learning_rate': 9.75041890773005e-07, 'epoch': 0.15134255492270138, 'num_input_tokens_seen': 1170210816, 'completed': '15.13% (558 / 3_687)', 'remaining time': '27:48:31', 'throughput': '8290.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:19:53,941 >> {'loss': 0.4236, 'grad_norm': 17.741085052490234, 'learning_rate': 9.749091470948643e-07, 'epoch': 0.15161377813940874, 'num_input_tokens_seen': 1172307968, 'completed': '15.16% (559 / 3_687)', 'remaining time': '27:47:43', 'throughput': '8995.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:20:25,033 >> {'loss': 0.4524, 'grad_norm': 13.735379219055176, 'learning_rate': 9.747760614660111e-07, 'epoch': 0.15188500135611607, 'num_input_tokens_seen': 1174405120, 'completed': '15.19% (560 / 3_687)', 'remaining time': '27:47:06', 'throughput': '8431.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:20:53,926 >> {'loss': 0.4694, 'grad_norm': 14.853401184082031, 'learning_rate': 9.746426339935477e-07, 'epoch': 0.15215622457282343, 'num_input_tokens_seen': 1176502272, 'completed': '15.22% (561 / 3_687)', 'remaining time': '27:46:17', 'throughput': '9073.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:21:24,472 >> {'loss': 0.32, 'grad_norm': 11.256040573120117, 'learning_rate': 9.745088647848515e-07, 'epoch': 0.1524274477895308, 'num_input_tokens_seen': 1178599424, 'completed': '15.24% (562 / 3_687)', 'remaining time': '27:45:37', 'throughput': '8581.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:21:53,659 >> {'loss': 0.7391, 'grad_norm': 21.761676788330078, 'learning_rate': 9.743747539475744e-07, 'epoch': 0.15269867100623813, 'num_input_tokens_seen': 1180696576, 'completed': '15.27% (563 / 3_687)', 'remaining time': '27:44:49', 'throughput': '8981.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:22:23,824 >> {'loss': 0.5598, 'grad_norm': 15.350510597229004, 'learning_rate': 9.74240301589644e-07, 'epoch': 0.1529698942229455, 'num_input_tokens_seen': 1182793728, 'completed': '15.30% (564 / 3_687)', 'remaining time': '27:44:07', 'throughput': '8690.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:22:53,701 >> {'loss': 0.4444, 'grad_norm': 12.007080078125, 'learning_rate': 9.741055078192626e-07, 'epoch': 0.15324111743965282, 'num_input_tokens_seen': 1184890880, 'completed': '15.32% (565 / 3_687)', 'remaining time': '27:43:24', 'throughput': '8773.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:23:23,447 >> {'loss': 0.4789, 'grad_norm': 12.715327262878418, 'learning_rate': 9.739703727449068e-07, 'epoch': 0.15351234065636019, 'num_input_tokens_seen': 1186988032, 'completed': '15.35% (566 / 3_687)', 'remaining time': '27:42:40', 'throughput': '8812.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:23:53,423 >> {'loss': 0.3289, 'grad_norm': 10.164214134216309, 'learning_rate': 9.738348964753283e-07, 'epoch': 0.15378356387306755, 'num_input_tokens_seen': 1189085184, 'completed': '15.38% (567 / 3_687)', 'remaining time': '27:41:57', 'throughput': '8745.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:24:23,561 >> {'loss': 0.4524, 'grad_norm': 15.66612720489502, 'learning_rate': 9.736990791195532e-07, 'epoch': 0.15405478708977488, 'num_input_tokens_seen': 1191182336, 'completed': '15.41% (568 / 3_687)', 'remaining time': '27:41:15', 'throughput': '8698.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:24:54,085 >> {'loss': 0.3797, 'grad_norm': 12.935588836669922, 'learning_rate': 9.735629207868824e-07, 'epoch': 0.15432601030648224, 'num_input_tokens_seen': 1193279488, 'completed': '15.43% (569 / 3_687)', 'remaining time': '27:40:35', 'throughput': '8587.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:25:24,155 >> {'loss': 0.5074, 'grad_norm': 14.650870323181152, 'learning_rate': 9.734264215868904e-07, 'epoch': 0.15459723352318958, 'num_input_tokens_seen': 1195376640, 'completed': '15.46% (570 / 3_687)', 'remaining time': '27:39:53', 'throughput': '8717.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:25:56,029 >> {'loss': 0.4658, 'grad_norm': 16.69452667236328, 'learning_rate': 9.73289581629427e-07, 'epoch': 0.15486845673989694, 'num_input_tokens_seen': 1197473792, 'completed': '15.49% (571 / 3_687)', 'remaining time': '27:39:20', 'throughput': '8224.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:26:26,101 >> {'loss': 0.7099, 'grad_norm': 19.8432674407959, 'learning_rate': 9.73152401024616e-07, 'epoch': 0.1551396799566043, 'num_input_tokens_seen': 1199570944, 'completed': '15.51% (572 / 3_687)', 'remaining time': '27:38:38', 'throughput': '8717.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:26:59,963 >> {'loss': 0.5642, 'grad_norm': 17.770639419555664, 'learning_rate': 9.730148798828543e-07, 'epoch': 0.15541090317331163, 'num_input_tokens_seen': 1201668096, 'completed': '15.54% (573 / 3_687)', 'remaining time': '27:38:16', 'throughput': '7741.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:27:28,592 >> {'loss': 0.7079, 'grad_norm': 20.076339721679688, 'learning_rate': 9.728770183148143e-07, 'epoch': 0.155682126390019, 'num_input_tokens_seen': 1203765248, 'completed': '15.57% (574 / 3_687)', 'remaining time': '27:37:27', 'throughput': '9156.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:27:58,262 >> {'loss': 0.721, 'grad_norm': 18.26179313659668, 'learning_rate': 9.727388164314415e-07, 'epoch': 0.15595334960672633, 'num_input_tokens_seen': 1205862400, 'completed': '15.60% (575 / 3_687)', 'remaining time': '27:36:42', 'throughput': '8835.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:28:29,567 >> {'loss': 0.5755, 'grad_norm': 21.209247589111328, 'learning_rate': 9.72600274343956e-07, 'epoch': 0.1562245728234337, 'num_input_tokens_seen': 1207959552, 'completed': '15.62% (576 / 3_687)', 'remaining time': '27:36:07', 'throughput': '8374.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:29:02,074 >> {'loss': 0.3086, 'grad_norm': 16.366296768188477, 'learning_rate': 9.724613921638506e-07, 'epoch': 0.15649579604014105, 'num_input_tokens_seen': 1210056704, 'completed': '15.65% (577 / 3_687)', 'remaining time': '27:35:38', 'throughput': '8064.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:29:31,396 >> {'loss': 0.5038, 'grad_norm': 16.17742919921875, 'learning_rate': 9.723221700028928e-07, 'epoch': 0.15676701925684838, 'num_input_tokens_seen': 1212153856, 'completed': '15.68% (578 / 3_687)', 'remaining time': '27:34:52', 'throughput': '8940.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:30:02,884 >> {'loss': 0.6945, 'grad_norm': 19.471908569335938, 'learning_rate': 9.72182607973123e-07, 'epoch': 0.15703824247355574, 'num_input_tokens_seen': 1214251008, 'completed': '15.70% (579 / 3_687)', 'remaining time': '27:34:18', 'throughput': '8325.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:30:33,219 >> {'loss': 0.6556, 'grad_norm': 23.296817779541016, 'learning_rate': 9.720427061868558e-07, 'epoch': 0.15730946569026308, 'num_input_tokens_seen': 1216348160, 'completed': '15.73% (580 / 3_687)', 'remaining time': '27:33:37', 'throughput': '8641.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:31:02,479 >> {'loss': 0.3241, 'grad_norm': 11.207878112792969, 'learning_rate': 9.71902464756678e-07, 'epoch': 0.15758068890697044, 'num_input_tokens_seen': 1218445312, 'completed': '15.76% (581 / 3_687)', 'remaining time': '27:32:51', 'throughput': '8959.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:31:31,563 >> {'loss': 0.5486, 'grad_norm': 19.07700538635254, 'learning_rate': 9.717618837954517e-07, 'epoch': 0.15785191212367777, 'num_input_tokens_seen': 1220542464, 'completed': '15.79% (582 / 3_687)', 'remaining time': '27:32:04', 'throughput': '9013.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:32:00,653 >> {'loss': 0.5149, 'grad_norm': 13.274455070495605, 'learning_rate': 9.716209634163102e-07, 'epoch': 0.15812313534038513, 'num_input_tokens_seen': 1222639616, 'completed': '15.81% (583 / 3_687)', 'remaining time': '27:31:17', 'throughput': '9011.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:32:32,710 >> {'loss': 0.4192, 'grad_norm': 13.982866287231445, 'learning_rate': 9.714797037326616e-07, 'epoch': 0.1583943585570925, 'num_input_tokens_seen': 1224736768, 'completed': '15.84% (584 / 3_687)', 'remaining time': '27:30:46', 'throughput': '8177.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:33:01,443 >> {'loss': 0.7549, 'grad_norm': 17.50905990600586, 'learning_rate': 9.713381048581855e-07, 'epoch': 0.15866558177379983, 'num_input_tokens_seen': 1226833920, 'completed': '15.87% (585 / 3_687)', 'remaining time': '27:29:57', 'throughput': '9123.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:33:33,937 >> {'loss': 0.4246, 'grad_norm': 12.565037727355957, 'learning_rate': 9.71196166906836e-07, 'epoch': 0.1589368049905072, 'num_input_tokens_seen': 1228931072, 'completed': '15.89% (586 / 3_687)', 'remaining time': '27:29:28', 'throughput': '8067.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:34:03,231 >> {'loss': 0.6191, 'grad_norm': 14.987252235412598, 'learning_rate': 9.71053889992839e-07, 'epoch': 0.15920802820721452, 'num_input_tokens_seen': 1231028224, 'completed': '15.92% (587 / 3_687)', 'remaining time': '27:28:42', 'throughput': '8948.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:34:34,718 >> {'loss': 0.4308, 'grad_norm': 11.8529691696167, 'learning_rate': 9.709112742306936e-07, 'epoch': 0.1594792514239219, 'num_input_tokens_seen': 1233125376, 'completed': '15.95% (588 / 3_687)', 'remaining time': '27:28:08', 'throughput': '8325.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:35:03,384 >> {'loss': 0.375, 'grad_norm': 11.237396240234375, 'learning_rate': 9.707683197351715e-07, 'epoch': 0.15975047464062925, 'num_input_tokens_seen': 1235222528, 'completed': '15.98% (589 / 3_687)', 'remaining time': '27:27:19', 'throughput': '9144.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:35:35,360 >> {'loss': 0.2555, 'grad_norm': 11.192814826965332, 'learning_rate': 9.706250266213173e-07, 'epoch': 0.16002169785733658, 'num_input_tokens_seen': 1237319680, 'completed': '16.00% (590 / 3_687)', 'remaining time': '27:26:47', 'throughput': '8198.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:36:05,063 >> {'loss': 0.3542, 'grad_norm': 11.35294246673584, 'learning_rate': 9.704813950044476e-07, 'epoch': 0.16029292107404394, 'num_input_tokens_seen': 1239416832, 'completed': '16.03% (591 / 3_687)', 'remaining time': '27:26:04', 'throughput': '8825.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:36:33,581 >> {'loss': 0.7244, 'grad_norm': 17.085115432739258, 'learning_rate': 9.703374250001516e-07, 'epoch': 0.16056414429075128, 'num_input_tokens_seen': 1241513984, 'completed': '16.06% (592 / 3_687)', 'remaining time': '27:25:14', 'throughput': '9192.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:37:01,546 >> {'loss': 0.6381, 'grad_norm': 15.55121898651123, 'learning_rate': 9.70193116724291e-07, 'epoch': 0.16083536750745864, 'num_input_tokens_seen': 1243611136, 'completed': '16.08% (593 / 3_687)', 'remaining time': '27:24:22', 'throughput': '9374.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:37:32,786 >> {'loss': 0.5695, 'grad_norm': 17.19528579711914, 'learning_rate': 9.700484702929996e-07, 'epoch': 0.161106590724166, 'num_input_tokens_seen': 1245708288, 'completed': '16.11% (594 / 3_687)', 'remaining time': '27:23:47', 'throughput': '8391.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:38:03,408 >> {'loss': 0.5896, 'grad_norm': 14.350486755371094, 'learning_rate': 9.699034858226834e-07, 'epoch': 0.16137781394087333, 'num_input_tokens_seen': 1247805440, 'completed': '16.14% (595 / 3_687)', 'remaining time': '27:23:08', 'throughput': '8560.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:38:33,838 >> {'loss': 0.3935, 'grad_norm': 11.893250465393066, 'learning_rate': 9.697581634300202e-07, 'epoch': 0.1616490371575807, 'num_input_tokens_seen': 1249902592, 'completed': '16.16% (596 / 3_687)', 'remaining time': '27:22:29', 'throughput': '8614.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:39:04,495 >> {'loss': 0.8572, 'grad_norm': 17.247047424316406, 'learning_rate': 9.6961250323196e-07, 'epoch': 0.16192026037428803, 'num_input_tokens_seen': 1251999744, 'completed': '16.19% (597 / 3_687)', 'remaining time': '27:21:51', 'throughput': '8550.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:39:33,093 >> {'loss': 0.8848, 'grad_norm': 22.222002029418945, 'learning_rate': 9.69466505345725e-07, 'epoch': 0.1621914835909954, 'num_input_tokens_seen': 1254096896, 'completed': '16.22% (598 / 3_687)', 'remaining time': '27:21:02', 'throughput': '9166.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:40:02,175 >> {'loss': 0.7104, 'grad_norm': 17.794937133789062, 'learning_rate': 9.69320169888808e-07, 'epoch': 0.16246270680770275, 'num_input_tokens_seen': 1256194048, 'completed': '16.25% (599 / 3_687)', 'remaining time': '27:20:16', 'throughput': '9013.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:40:32,522 >> {'loss': 0.5435, 'grad_norm': 13.776861190795898, 'learning_rate': 9.691734969789746e-07, 'epoch': 0.16273393002441008, 'num_input_tokens_seen': 1258291200, 'completed': '16.27% (600 / 3_687)', 'remaining time': '27:19:36', 'throughput': '8638.35', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2025-01-01 16:40:57,435 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-600 +[INFO|configuration_utils.py:472] 2025-01-01 16:40:57,438 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-600/config.json +[INFO|configuration_utils.py:807] 2025-01-01 16:40:57,440 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-600/generation_config.json +[INFO|modeling_utils.py:2807] 2025-01-01 16:41:56,467 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-600/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2025-01-01 16:41:56,470 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-600/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2025-01-01 16:41:56,471 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-600/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2025-01-01 16:45:32,314 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 1200, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2025-01-01 16:46:02,401 >> {'loss': 0.4268, 'grad_norm': 15.114326477050781, 'learning_rate': 9.690264867342618e-07, 'epoch': 0.16300515324111745, 'num_input_tokens_seen': 1260388352, 'completed': '16.30% (601 / 3_687)', 'remaining time': '27:44:34', 'throughput': '794.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:46:32,791 >> {'loss': 0.5672, 'grad_norm': 14.888233184814453, 'learning_rate': 9.688791392729775e-07, 'epoch': 0.16327637645782478, 'num_input_tokens_seen': 1262485504, 'completed': '16.33% (602 / 3_687)', 'remaining time': '27:43:52', 'throughput': '8626.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:47:02,426 >> {'loss': 0.4089, 'grad_norm': 12.982342720031738, 'learning_rate': 9.687314547137016e-07, 'epoch': 0.16354759967453214, 'num_input_tokens_seen': 1264582656, 'completed': '16.35% (603 / 3_687)', 'remaining time': '27:43:05', 'throughput': '8845.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:47:34,442 >> {'loss': 0.2905, 'grad_norm': 11.086604118347168, 'learning_rate': 9.685834331752846e-07, 'epoch': 0.1638188228912395, 'num_input_tokens_seen': 1266679808, 'completed': '16.38% (604 / 3_687)', 'remaining time': '27:42:31', 'throughput': '8188.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:48:00,805 >> {'loss': 0.9503, 'grad_norm': 20.390928268432617, 'learning_rate': 9.684350747768492e-07, 'epoch': 0.16409004610794684, 'num_input_tokens_seen': 1268776960, 'completed': '16.41% (605 / 3_687)', 'remaining time': '27:41:28', 'throughput': '9943.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:48:31,315 >> {'loss': 0.5335, 'grad_norm': 35.44593048095703, 'learning_rate': 9.68286379637788e-07, 'epoch': 0.1643612693246542, 'num_input_tokens_seen': 1270874112, 'completed': '16.44% (606 / 3_687)', 'remaining time': '27:40:47', 'throughput': '8592.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:49:04,480 >> {'loss': 0.3713, 'grad_norm': 11.213769912719727, 'learning_rate': 9.681373478777654e-07, 'epoch': 0.16463249254136153, 'num_input_tokens_seen': 1272971264, 'completed': '16.46% (607 / 3_687)', 'remaining time': '27:40:19', 'throughput': '7904.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:49:36,025 >> {'loss': 0.4049, 'grad_norm': 11.05787181854248, 'learning_rate': 9.679879796167166e-07, 'epoch': 0.1649037157580689, 'num_input_tokens_seen': 1275068416, 'completed': '16.49% (608 / 3_687)', 'remaining time': '27:39:42', 'throughput': '8310.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:50:07,391 >> {'loss': 0.5591, 'grad_norm': 15.489423751831055, 'learning_rate': 9.678382749748477e-07, 'epoch': 0.16517493897477625, 'num_input_tokens_seen': 1277165568, 'completed': '16.52% (609 / 3_687)', 'remaining time': '27:39:05', 'throughput': '8357.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:50:38,921 >> {'loss': 0.592, 'grad_norm': 16.684755325317383, 'learning_rate': 9.676882340726345e-07, 'epoch': 0.1654461621914836, 'num_input_tokens_seen': 1279262720, 'completed': '16.54% (610 / 3_687)', 'remaining time': '27:38:29', 'throughput': '8314.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:51:09,031 >> {'loss': 0.6475, 'grad_norm': 17.940027236938477, 'learning_rate': 9.675378570308253e-07, 'epoch': 0.16571738540819095, 'num_input_tokens_seen': 1281359872, 'completed': '16.57% (611 / 3_687)', 'remaining time': '27:37:45', 'throughput': '8706.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:51:41,435 >> {'loss': 0.3884, 'grad_norm': 11.783378601074219, 'learning_rate': 9.673871439704369e-07, 'epoch': 0.16598860862489828, 'num_input_tokens_seen': 1283457024, 'completed': '16.60% (612 / 3_687)', 'remaining time': '27:37:13', 'throughput': '8089.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:52:12,461 >> {'loss': 0.5195, 'grad_norm': 16.390104293823242, 'learning_rate': 9.672360950127578e-07, 'epoch': 0.16625983184160564, 'num_input_tokens_seen': 1285554176, 'completed': '16.63% (613 / 3_687)', 'remaining time': '27:36:34', 'throughput': '8449.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:52:45,300 >> {'loss': 0.3534, 'grad_norm': 15.546297073364258, 'learning_rate': 9.670847102793464e-07, 'epoch': 0.166531055058313, 'num_input_tokens_seen': 1287651328, 'completed': '16.65% (614 / 3_687)', 'remaining time': '27:36:04', 'throughput': '7982.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:53:16,349 >> {'loss': 0.5498, 'grad_norm': 15.699833869934082, 'learning_rate': 9.669329898920317e-07, 'epoch': 0.16680227827502034, 'num_input_tokens_seen': 1289748480, 'completed': '16.68% (615 / 3_687)', 'remaining time': '27:35:25', 'throughput': '8442.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:53:48,301 >> {'loss': 0.6282, 'grad_norm': 19.23993682861328, 'learning_rate': 9.66780933972912e-07, 'epoch': 0.1670735014917277, 'num_input_tokens_seen': 1291845632, 'completed': '16.71% (616 / 3_687)', 'remaining time': '27:34:51', 'throughput': '8204.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:54:17,692 >> {'loss': 0.2954, 'grad_norm': 9.65346908569336, 'learning_rate': 9.666285426443564e-07, 'epoch': 0.16734472470843503, 'num_input_tokens_seen': 1293942784, 'completed': '16.73% (617 / 3_687)', 'remaining time': '27:34:04', 'throughput': '8919.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:54:50,658 >> {'loss': 0.444, 'grad_norm': 11.71711254119873, 'learning_rate': 9.664758160290036e-07, 'epoch': 0.1676159479251424, 'num_input_tokens_seen': 1296039936, 'completed': '16.76% (618 / 3_687)', 'remaining time': '27:33:35', 'throughput': '7951.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:55:20,880 >> {'loss': 0.6955, 'grad_norm': 20.66088104248047, 'learning_rate': 9.66322754249762e-07, 'epoch': 0.16788717114184976, 'num_input_tokens_seen': 1298137088, 'completed': '16.79% (619 / 3_687)', 'remaining time': '27:32:52', 'throughput': '8674.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:55:51,339 >> {'loss': 0.8379, 'grad_norm': 20.292829513549805, 'learning_rate': 9.661693574298102e-07, 'epoch': 0.1681583943585571, 'num_input_tokens_seen': 1300234240, 'completed': '16.82% (620 / 3_687)', 'remaining time': '27:32:11', 'throughput': '8606.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:56:24,838 >> {'loss': 0.3796, 'grad_norm': 12.26496410369873, 'learning_rate': 9.66015625692596e-07, 'epoch': 0.16842961757526445, 'num_input_tokens_seen': 1302331392, 'completed': '16.84% (621 / 3_687)', 'remaining time': '27:31:44', 'throughput': '7825.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:56:56,097 >> {'loss': 0.2908, 'grad_norm': 13.123128890991211, 'learning_rate': 9.658615591618366e-07, 'epoch': 0.16870084079197178, 'num_input_tokens_seen': 1304428544, 'completed': '16.87% (622 / 3_687)', 'remaining time': '27:31:07', 'throughput': '8386.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:57:27,218 >> {'loss': 0.4747, 'grad_norm': 13.454907417297363, 'learning_rate': 9.657071579615191e-07, 'epoch': 0.16897206400867915, 'num_input_tokens_seen': 1306525696, 'completed': '16.90% (623 / 3_687)', 'remaining time': '27:30:29', 'throughput': '8423.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:57:59,590 >> {'loss': 0.745, 'grad_norm': 17.779613494873047, 'learning_rate': 9.655524222159e-07, 'epoch': 0.16924328722538648, 'num_input_tokens_seen': 1308622848, 'completed': '16.92% (624 / 3_687)', 'remaining time': '27:29:56', 'throughput': '8097.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:58:28,877 >> {'loss': 0.6889, 'grad_norm': 18.326885223388672, 'learning_rate': 9.653973520495042e-07, 'epoch': 0.16951451044209384, 'num_input_tokens_seen': 1310720000, 'completed': '16.95% (625 / 3_687)', 'remaining time': '27:29:09', 'throughput': '8951.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:59:00,386 >> {'loss': 0.4917, 'grad_norm': 15.276704788208008, 'learning_rate': 9.652419475871267e-07, 'epoch': 0.1697857336588012, 'num_input_tokens_seen': 1312817152, 'completed': '16.98% (626 / 3_687)', 'remaining time': '27:28:33', 'throughput': '8319.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 16:59:32,387 >> {'loss': 0.4999, 'grad_norm': 15.219459533691406, 'learning_rate': 9.650862089538307e-07, 'epoch': 0.17005695687550854, 'num_input_tokens_seen': 1314914304, 'completed': '17.01% (627 / 3_687)', 'remaining time': '27:27:59', 'throughput': '8191.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:00:02,480 >> {'loss': 0.5888, 'grad_norm': 16.05649757385254, 'learning_rate': 9.64930136274949e-07, 'epoch': 0.1703281800922159, 'num_input_tokens_seen': 1317011456, 'completed': '17.03% (628 / 3_687)', 'remaining time': '27:27:16', 'throughput': '8711.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:00:32,134 >> {'loss': 0.6463, 'grad_norm': 16.75012969970703, 'learning_rate': 9.647737296760828e-07, 'epoch': 0.17059940330892323, 'num_input_tokens_seen': 1319108608, 'completed': '17.06% (629 / 3_687)', 'remaining time': '27:26:31', 'throughput': '8840.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:01:03,937 >> {'loss': 0.4548, 'grad_norm': 14.331846237182617, 'learning_rate': 9.646169892831025e-07, 'epoch': 0.1708706265256306, 'num_input_tokens_seen': 1321205760, 'completed': '17.09% (630 / 3_687)', 'remaining time': '27:25:56', 'throughput': '8242.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:01:34,501 >> {'loss': 0.5452, 'grad_norm': 14.530692100524902, 'learning_rate': 9.644599152221465e-07, 'epoch': 0.17114184974233795, 'num_input_tokens_seen': 1323302912, 'completed': '17.11% (631 / 3_687)', 'remaining time': '27:25:15', 'throughput': '8577.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:02:03,681 >> {'loss': 0.599, 'grad_norm': 13.921281814575195, 'learning_rate': 9.643025076196219e-07, 'epoch': 0.1714130729590453, 'num_input_tokens_seen': 1325400064, 'completed': '17.14% (632 / 3_687)', 'remaining time': '27:24:28', 'throughput': '8983.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:02:36,743 >> {'loss': 0.5125, 'grad_norm': 14.689400672912598, 'learning_rate': 9.641447666022048e-07, 'epoch': 0.17168429617575265, 'num_input_tokens_seen': 1327497216, 'completed': '17.17% (633 / 3_687)', 'remaining time': '27:23:59', 'throughput': '7928.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:03:09,822 >> {'loss': 0.7141, 'grad_norm': 21.08767318725586, 'learning_rate': 9.639866922968387e-07, 'epoch': 0.17195551939245998, 'num_input_tokens_seen': 1329594368, 'completed': '17.20% (634 / 3_687)', 'remaining time': '27:23:31', 'throughput': '7924.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:03:40,689 >> {'loss': 0.7668, 'grad_norm': 21.691171646118164, 'learning_rate': 9.638282848307361e-07, 'epoch': 0.17222674260916734, 'num_input_tokens_seen': 1331691520, 'completed': '17.22% (635 / 3_687)', 'remaining time': '27:22:52', 'throughput': '8492.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:04:11,640 >> {'loss': 0.6828, 'grad_norm': 18.770170211791992, 'learning_rate': 9.636695443313773e-07, 'epoch': 0.1724979658258747, 'num_input_tokens_seen': 1333788672, 'completed': '17.25% (636 / 3_687)', 'remaining time': '27:22:13', 'throughput': '8469.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:04:39,405 >> {'loss': 0.8134, 'grad_norm': 18.908126831054688, 'learning_rate': 9.635104709265103e-07, 'epoch': 0.17276918904258204, 'num_input_tokens_seen': 1335885824, 'completed': '17.28% (637 / 3_687)', 'remaining time': '27:21:19', 'throughput': '9441.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:05:09,181 >> {'loss': 0.3847, 'grad_norm': 11.843361854553223, 'learning_rate': 9.633510647441518e-07, 'epoch': 0.1730404122592894, 'num_input_tokens_seen': 1337982976, 'completed': '17.30% (638 / 3_687)', 'remaining time': '27:20:35', 'throughput': '8803.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:05:36,827 >> {'loss': 0.9532, 'grad_norm': 19.700693130493164, 'learning_rate': 9.631913259125854e-07, 'epoch': 0.17331163547599673, 'num_input_tokens_seen': 1340080128, 'completed': '17.33% (639 / 3_687)', 'remaining time': '27:19:40', 'throughput': '9482.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:06:07,721 >> {'loss': 0.3257, 'grad_norm': 10.508553504943848, 'learning_rate': 9.630312545603631e-07, 'epoch': 0.1735828586927041, 'num_input_tokens_seen': 1342177280, 'completed': '17.36% (640 / 3_687)', 'remaining time': '27:19:01', 'throughput': '8485.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:06:40,434 >> {'loss': 0.7283, 'grad_norm': 19.769977569580078, 'learning_rate': 9.628708508163041e-07, 'epoch': 0.17385408190941146, 'num_input_tokens_seen': 1344274432, 'completed': '17.39% (641 / 3_687)', 'remaining time': '27:18:31', 'throughput': '8013.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:07:12,095 >> {'loss': 0.3603, 'grad_norm': 10.588939666748047, 'learning_rate': 9.627101148094952e-07, 'epoch': 0.1741253051261188, 'num_input_tokens_seen': 1346371584, 'completed': '17.41% (642 / 3_687)', 'remaining time': '27:17:56', 'throughput': '8279.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:07:39,377 >> {'loss': 0.7399, 'grad_norm': 16.81378936767578, 'learning_rate': 9.625490466692906e-07, 'epoch': 0.17439652834282615, 'num_input_tokens_seen': 1348468736, 'completed': '17.44% (643 / 3_687)', 'remaining time': '27:17:00', 'throughput': '9608.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:08:12,072 >> {'loss': 0.4786, 'grad_norm': 13.483939170837402, 'learning_rate': 9.623876465253122e-07, 'epoch': 0.17466775155953349, 'num_input_tokens_seen': 1350565888, 'completed': '17.47% (644 / 3_687)', 'remaining time': '27:16:30', 'throughput': '8017.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:08:45,011 >> {'loss': 0.4953, 'grad_norm': 16.27707290649414, 'learning_rate': 9.622259145074482e-07, 'epoch': 0.17493897477624085, 'num_input_tokens_seen': 1352663040, 'completed': '17.49% (645 / 3_687)', 'remaining time': '27:16:01', 'throughput': '7958.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:09:16,268 >> {'loss': 0.6285, 'grad_norm': 15.7435941696167, 'learning_rate': 9.620638507458547e-07, 'epoch': 0.1752101979929482, 'num_input_tokens_seen': 1354760192, 'completed': '17.52% (646 / 3_687)', 'remaining time': '27:15:24', 'throughput': '8386.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:09:43,676 >> {'loss': 0.8913, 'grad_norm': 20.336198806762695, 'learning_rate': 9.619014553709542e-07, 'epoch': 0.17548142120965554, 'num_input_tokens_seen': 1356857344, 'completed': '17.55% (647 / 3_687)', 'remaining time': '27:14:29', 'throughput': '9564.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:10:15,612 >> {'loss': 0.9168, 'grad_norm': 25.84923553466797, 'learning_rate': 9.617387285134364e-07, 'epoch': 0.1757526444263629, 'num_input_tokens_seen': 1358954496, 'completed': '17.58% (648 / 3_687)', 'remaining time': '27:13:55', 'throughput': '8208.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:10:46,392 >> {'loss': 0.4383, 'grad_norm': 12.170639991760254, 'learning_rate': 9.615756703042575e-07, 'epoch': 0.17602386764307024, 'num_input_tokens_seen': 1361051648, 'completed': '17.60% (649 / 3_687)', 'remaining time': '27:13:16', 'throughput': '8516.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:11:17,234 >> {'loss': 0.4632, 'grad_norm': 16.723989486694336, 'learning_rate': 9.61412280874641e-07, 'epoch': 0.1762950908597776, 'num_input_tokens_seen': 1363148800, 'completed': '17.63% (650 / 3_687)', 'remaining time': '27:12:37', 'throughput': '8499.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:11:45,824 >> {'loss': 0.8342, 'grad_norm': 16.861356735229492, 'learning_rate': 9.612485603560763e-07, 'epoch': 0.17656631407648496, 'num_input_tokens_seen': 1365245952, 'completed': '17.66% (651 / 3_687)', 'remaining time': '27:11:47', 'throughput': '9169.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:12:15,787 >> {'loss': 0.5916, 'grad_norm': 19.289173126220703, 'learning_rate': 9.610845088803194e-07, 'epoch': 0.1768375372931923, 'num_input_tokens_seen': 1367343104, 'completed': '17.68% (652 / 3_687)', 'remaining time': '27:11:04', 'throughput': '8748.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:12:44,252 >> {'loss': 0.5876, 'grad_norm': 17.8167781829834, 'learning_rate': 9.609201265793927e-07, 'epoch': 0.17710876050989965, 'num_input_tokens_seen': 1369440256, 'completed': '17.71% (653 / 3_687)', 'remaining time': '27:10:15', 'throughput': '9209.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:13:13,236 >> {'loss': 0.524, 'grad_norm': 13.8448486328125, 'learning_rate': 9.607554135855847e-07, 'epoch': 0.177379983726607, 'num_input_tokens_seen': 1371537408, 'completed': '17.74% (654 / 3_687)', 'remaining time': '27:09:27', 'throughput': '9044.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:13:43,306 >> {'loss': 0.4775, 'grad_norm': 14.017175674438477, 'learning_rate': 9.605903700314503e-07, 'epoch': 0.17765120694331435, 'num_input_tokens_seen': 1373634560, 'completed': '17.77% (655 / 3_687)', 'remaining time': '27:08:45', 'throughput': '8718.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:14:14,684 >> {'loss': 0.359, 'grad_norm': 11.310769081115723, 'learning_rate': 9.604249960498102e-07, 'epoch': 0.1779224301600217, 'num_input_tokens_seen': 1375731712, 'completed': '17.79% (656 / 3_687)', 'remaining time': '27:08:09', 'throughput': '8354.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:14:46,271 >> {'loss': 0.4158, 'grad_norm': 13.998455047607422, 'learning_rate': 9.602592917737512e-07, 'epoch': 0.17819365337672904, 'num_input_tokens_seen': 1377828864, 'completed': '17.82% (657 / 3_687)', 'remaining time': '27:07:34', 'throughput': '8299.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:15:13,488 >> {'loss': 0.8973, 'grad_norm': 21.207416534423828, 'learning_rate': 9.600932573366254e-07, 'epoch': 0.1784648765934364, 'num_input_tokens_seen': 1379926016, 'completed': '17.85% (658 / 3_687)', 'remaining time': '27:06:38', 'throughput': '9631.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:15:43,441 >> {'loss': 0.3974, 'grad_norm': 12.346809387207031, 'learning_rate': 9.599268928720518e-07, 'epoch': 0.17873609981014374, 'num_input_tokens_seen': 1382023168, 'completed': '17.87% (659 / 3_687)', 'remaining time': '27:05:56', 'throughput': '8751.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:16:13,687 >> {'loss': 0.3364, 'grad_norm': 16.258766174316406, 'learning_rate': 9.597601985139132e-07, 'epoch': 0.1790073230268511, 'num_input_tokens_seen': 1384120320, 'completed': '17.90% (660 / 3_687)', 'remaining time': '27:05:14', 'throughput': '8667.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:16:43,536 >> {'loss': 0.5018, 'grad_norm': 16.505359649658203, 'learning_rate': 9.595931743963596e-07, 'epoch': 0.17927854624355846, 'num_input_tokens_seen': 1386217472, 'completed': '17.93% (661 / 3_687)', 'remaining time': '27:04:31', 'throughput': '8782.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:17:15,531 >> {'loss': 0.4393, 'grad_norm': 15.472549438476562, 'learning_rate': 9.594258206538054e-07, 'epoch': 0.1795497694602658, 'num_input_tokens_seen': 1388314624, 'completed': '17.95% (662 / 3_687)', 'remaining time': '27:03:58', 'throughput': '8193.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:17:45,453 >> {'loss': 0.3907, 'grad_norm': 12.654108047485352, 'learning_rate': 9.592581374209306e-07, 'epoch': 0.17982099267697316, 'num_input_tokens_seen': 1390411776, 'completed': '17.98% (663 / 3_687)', 'remaining time': '27:03:16', 'throughput': '8760.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:18:18,102 >> {'loss': 0.5991, 'grad_norm': 15.311179161071777, 'learning_rate': 9.590901248326802e-07, 'epoch': 0.1800922158936805, 'num_input_tokens_seen': 1392508928, 'completed': '18.01% (664 / 3_687)', 'remaining time': '27:02:45', 'throughput': '8029.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:18:49,541 >> {'loss': 0.3853, 'grad_norm': 13.300477981567383, 'learning_rate': 9.589217830242645e-07, 'epoch': 0.18036343911038785, 'num_input_tokens_seen': 1394606080, 'completed': '18.04% (665 / 3_687)', 'remaining time': '27:02:10', 'throughput': '8338.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:19:20,513 >> {'loss': 0.3291, 'grad_norm': 14.14307975769043, 'learning_rate': 9.587531121311582e-07, 'epoch': 0.18063466232709519, 'num_input_tokens_seen': 1396703232, 'completed': '18.06% (666 / 3_687)', 'remaining time': '27:01:32', 'throughput': '8463.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:19:50,974 >> {'loss': 0.8024, 'grad_norm': 19.638458251953125, 'learning_rate': 9.585841122891016e-07, 'epoch': 0.18090588554380255, 'num_input_tokens_seen': 1398800384, 'completed': '18.09% (667 / 3_687)', 'remaining time': '27:00:52', 'throughput': '8605.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:20:26,762 >> {'loss': 0.6279, 'grad_norm': 18.304893493652344, 'learning_rate': 9.584147836340992e-07, 'epoch': 0.1811771087605099, 'num_input_tokens_seen': 1400897536, 'completed': '18.12% (668 / 3_687)', 'remaining time': '27:00:36', 'throughput': '7324.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:20:57,463 >> {'loss': 0.5066, 'grad_norm': 13.643768310546875, 'learning_rate': 9.582451263024202e-07, 'epoch': 0.18144833197721724, 'num_input_tokens_seen': 1402994688, 'completed': '18.14% (669 / 3_687)', 'remaining time': '26:59:57', 'throughput': '8538.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:21:23,435 >> {'loss': 0.7491, 'grad_norm': 17.128938674926758, 'learning_rate': 9.580751404305985e-07, 'epoch': 0.1817195551939246, 'num_input_tokens_seen': 1405091840, 'completed': '18.17% (670 / 3_687)', 'remaining time': '26:58:56', 'throughput': '10093.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:21:51,019 >> {'loss': 0.8889, 'grad_norm': 18.64154052734375, 'learning_rate': 9.579048261554321e-07, 'epoch': 0.18199077841063194, 'num_input_tokens_seen': 1407188992, 'completed': '18.20% (671 / 3_687)', 'remaining time': '26:58:04', 'throughput': '9503.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:22:21,189 >> {'loss': 0.4579, 'grad_norm': 14.319369316101074, 'learning_rate': 9.577341836139837e-07, 'epoch': 0.1822620016273393, 'num_input_tokens_seen': 1409286144, 'completed': '18.23% (672 / 3_687)', 'remaining time': '26:57:22', 'throughput': '8688.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:22:52,532 >> {'loss': 0.6158, 'grad_norm': 15.52761459350586, 'learning_rate': 9.575632129435796e-07, 'epoch': 0.18253322484404666, 'num_input_tokens_seen': 1411383296, 'completed': '18.25% (673 / 3_687)', 'remaining time': '26:56:46', 'throughput': '8363.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:23:23,778 >> {'loss': 0.3947, 'grad_norm': 11.78895092010498, 'learning_rate': 9.573919142818109e-07, 'epoch': 0.182804448060754, 'num_input_tokens_seen': 1413480448, 'completed': '18.28% (674 / 3_687)', 'remaining time': '26:56:10', 'throughput': '8389.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:23:53,139 >> {'loss': 0.6729, 'grad_norm': 17.20797348022461, 'learning_rate': 9.572202877665317e-07, 'epoch': 0.18307567127746135, 'num_input_tokens_seen': 1415577600, 'completed': '18.31% (675 / 3_687)', 'remaining time': '26:55:25', 'throughput': '8928.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:24:23,641 >> {'loss': 0.2969, 'grad_norm': 10.157732009887695, 'learning_rate': 9.57048333535861e-07, 'epoch': 0.1833468944941687, 'num_input_tokens_seen': 1417674752, 'completed': '18.33% (676 / 3_687)', 'remaining time': '26:54:46', 'throughput': '8594.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:24:52,881 >> {'loss': 0.9544, 'grad_norm': 20.86275863647461, 'learning_rate': 9.568760517281808e-07, 'epoch': 0.18361811771087605, 'num_input_tokens_seen': 1419771904, 'completed': '18.36% (677 / 3_687)', 'remaining time': '26:54:00', 'throughput': '8965.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:25:23,064 >> {'loss': 0.3624, 'grad_norm': 14.223817825317383, 'learning_rate': 9.56703442482137e-07, 'epoch': 0.1838893409275834, 'num_input_tokens_seen': 1421869056, 'completed': '18.39% (678 / 3_687)', 'remaining time': '26:53:19', 'throughput': '8685.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:25:52,029 >> {'loss': 0.8115, 'grad_norm': 16.383438110351562, 'learning_rate': 9.565305059366385e-07, 'epoch': 0.18416056414429074, 'num_input_tokens_seen': 1423966208, 'completed': '18.42% (679 / 3_687)', 'remaining time': '26:52:33', 'throughput': '9050.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:26:22,848 >> {'loss': 0.3638, 'grad_norm': 12.147848129272461, 'learning_rate': 9.563572422308588e-07, 'epoch': 0.1844317873609981, 'num_input_tokens_seen': 1426063360, 'completed': '18.44% (680 / 3_687)', 'remaining time': '26:51:55', 'throughput': '8505.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:26:53,179 >> {'loss': 0.5524, 'grad_norm': 15.152525901794434, 'learning_rate': 9.561836515042336e-07, 'epoch': 0.18470301057770544, 'num_input_tokens_seen': 1428160512, 'completed': '18.47% (681 / 3_687)', 'remaining time': '26:51:15', 'throughput': '8642.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:27:22,395 >> {'loss': 0.4642, 'grad_norm': 13.729914665222168, 'learning_rate': 9.56009733896462e-07, 'epoch': 0.1849742337944128, 'num_input_tokens_seen': 1430257664, 'completed': '18.50% (682 / 3_687)', 'remaining time': '26:50:29', 'throughput': '8972.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:27:54,435 >> {'loss': 0.6551, 'grad_norm': 16.161523818969727, 'learning_rate': 9.558354895475065e-07, 'epoch': 0.18524545701112016, 'num_input_tokens_seen': 1432354816, 'completed': '18.52% (683 / 3_687)', 'remaining time': '26:49:57', 'throughput': '8181.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:28:25,063 >> {'loss': 0.9867, 'grad_norm': 22.390674591064453, 'learning_rate': 9.55660918597592e-07, 'epoch': 0.1855166802278275, 'num_input_tokens_seen': 1434451968, 'completed': '18.55% (684 / 3_687)', 'remaining time': '26:49:18', 'throughput': '8558.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:28:58,358 >> {'loss': 0.6112, 'grad_norm': 17.674570083618164, 'learning_rate': 9.55486021187207e-07, 'epoch': 0.18578790344453486, 'num_input_tokens_seen': 1436549120, 'completed': '18.58% (685 / 3_687)', 'remaining time': '26:48:51', 'throughput': '7873.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:29:27,737 >> {'loss': 0.3261, 'grad_norm': 11.00310230255127, 'learning_rate': 9.553107974571018e-07, 'epoch': 0.1860591266612422, 'num_input_tokens_seen': 1438646272, 'completed': '18.61% (686 / 3_687)', 'remaining time': '26:48:06', 'throughput': '8922.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:29:59,904 >> {'loss': 0.7754, 'grad_norm': 18.365558624267578, 'learning_rate': 9.551352475482902e-07, 'epoch': 0.18633034987794955, 'num_input_tokens_seen': 1440743424, 'completed': '18.63% (687 / 3_687)', 'remaining time': '26:47:34', 'throughput': '8149.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:30:30,679 >> {'loss': 0.4411, 'grad_norm': 13.15377140045166, 'learning_rate': 9.549593716020478e-07, 'epoch': 0.18660157309465691, 'num_input_tokens_seen': 1442840576, 'completed': '18.66% (688 / 3_687)', 'remaining time': '26:46:56', 'throughput': '8518.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:31:00,085 >> {'loss': 0.4043, 'grad_norm': 11.530652046203613, 'learning_rate': 9.54783169759913e-07, 'epoch': 0.18687279631136425, 'num_input_tokens_seen': 1444937728, 'completed': '18.69% (689 / 3_687)', 'remaining time': '26:46:12', 'throughput': '8914.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:31:30,524 >> {'loss': 0.389, 'grad_norm': 14.537562370300293, 'learning_rate': 9.546066421636867e-07, 'epoch': 0.1871440195280716, 'num_input_tokens_seen': 1447034880, 'completed': '18.71% (690 / 3_687)', 'remaining time': '26:45:33', 'throughput': '8612.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:32:05,576 >> {'loss': 0.2483, 'grad_norm': 9.434741020202637, 'learning_rate': 9.54429788955431e-07, 'epoch': 0.18741524274477894, 'num_input_tokens_seen': 1449132032, 'completed': '18.74% (691 / 3_687)', 'remaining time': '26:45:13', 'throughput': '7478.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:32:41,502 >> {'loss': 0.5747, 'grad_norm': 15.039995193481445, 'learning_rate': 9.542526102774704e-07, 'epoch': 0.1876864659614863, 'num_input_tokens_seen': 1451229184, 'completed': '18.77% (692 / 3_687)', 'remaining time': '26:44:57', 'throughput': '7296.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:33:10,183 >> {'loss': 0.7014, 'grad_norm': 16.6597900390625, 'learning_rate': 9.540751062723923e-07, 'epoch': 0.18795768917819367, 'num_input_tokens_seen': 1453326336, 'completed': '18.80% (693 / 3_687)', 'remaining time': '26:44:10', 'throughput': '9139.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:33:39,469 >> {'loss': 0.7324, 'grad_norm': 17.40535545349121, 'learning_rate': 9.53897277083045e-07, 'epoch': 0.188228912394901, 'num_input_tokens_seen': 1455423488, 'completed': '18.82% (694 / 3_687)', 'remaining time': '26:43:26', 'throughput': '8951.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:34:10,499 >> {'loss': 0.4232, 'grad_norm': 14.920904159545898, 'learning_rate': 9.537191228525382e-07, 'epoch': 0.18850013561160836, 'num_input_tokens_seen': 1457520640, 'completed': '18.85% (695 / 3_687)', 'remaining time': '26:42:49', 'throughput': '8448.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:34:42,550 >> {'loss': 0.5262, 'grad_norm': 16.280458450317383, 'learning_rate': 9.535406437242444e-07, 'epoch': 0.1887713588283157, 'num_input_tokens_seen': 1459617792, 'completed': '18.88% (696 / 3_687)', 'remaining time': '26:42:16', 'throughput': '8178.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:35:15,088 >> {'loss': 0.534, 'grad_norm': 13.120879173278809, 'learning_rate': 9.533618398417962e-07, 'epoch': 0.18904258204502306, 'num_input_tokens_seen': 1461714944, 'completed': '18.90% (697 / 3_687)', 'remaining time': '26:41:46', 'throughput': '8056.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:35:45,167 >> {'loss': 0.6104, 'grad_norm': 16.71864891052246, 'learning_rate': 9.531827113490883e-07, 'epoch': 0.18931380526173042, 'num_input_tokens_seen': 1463812096, 'completed': '18.93% (698 / 3_687)', 'remaining time': '26:41:05', 'throughput': '8715.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:36:16,856 >> {'loss': 0.5911, 'grad_norm': 14.800333023071289, 'learning_rate': 9.530032583902767e-07, 'epoch': 0.18958502847843775, 'num_input_tokens_seen': 1465909248, 'completed': '18.96% (699 / 3_687)', 'remaining time': '26:40:31', 'throughput': '8272.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:36:48,642 >> {'loss': 0.6818, 'grad_norm': 15.830343246459961, 'learning_rate': 9.528234811097781e-07, 'epoch': 0.1898562516951451, 'num_input_tokens_seen': 1468006400, 'completed': '18.99% (700 / 3_687)', 'remaining time': '26:39:57', 'throughput': '8247.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:37:19,915 >> {'loss': 0.6553, 'grad_norm': 20.223512649536133, 'learning_rate': 9.526433796522702e-07, 'epoch': 0.19012747491185245, 'num_input_tokens_seen': 1470103552, 'completed': '19.01% (701 / 3_687)', 'remaining time': '26:39:21', 'throughput': '8382.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:37:51,730 >> {'loss': 0.4537, 'grad_norm': 12.725762367248535, 'learning_rate': 9.524629541626925e-07, 'epoch': 0.1903986981285598, 'num_input_tokens_seen': 1472200704, 'completed': '19.04% (702 / 3_687)', 'remaining time': '26:38:48', 'throughput': '8239.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:38:20,624 >> {'loss': 0.6253, 'grad_norm': 18.15487289428711, 'learning_rate': 9.522822047862438e-07, 'epoch': 0.19066992134526717, 'num_input_tokens_seen': 1474297856, 'completed': '19.07% (703 / 3_687)', 'remaining time': '26:38:02', 'throughput': '9072.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:38:51,209 >> {'loss': 0.6778, 'grad_norm': 17.46477699279785, 'learning_rate': 9.521011316683849e-07, 'epoch': 0.1909411445619745, 'num_input_tokens_seen': 1476395008, 'completed': '19.09% (704 / 3_687)', 'remaining time': '26:37:23', 'throughput': '8570.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:39:21,740 >> {'loss': 0.6406, 'grad_norm': 16.60508155822754, 'learning_rate': 9.519197349548364e-07, 'epoch': 0.19121236777868186, 'num_input_tokens_seen': 1478492160, 'completed': '19.12% (705 / 3_687)', 'remaining time': '26:36:44', 'throughput': '8586.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:39:52,863 >> {'loss': 0.4449, 'grad_norm': 14.466193199157715, 'learning_rate': 9.517380147915791e-07, 'epoch': 0.1914835909953892, 'num_input_tokens_seen': 1480589312, 'completed': '19.15% (706 / 3_687)', 'remaining time': '26:36:08', 'throughput': '8422.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:40:24,381 >> {'loss': 0.6814, 'grad_norm': 16.949872970581055, 'learning_rate': 9.515559713248549e-07, 'epoch': 0.19175481421209656, 'num_input_tokens_seen': 1482686464, 'completed': '19.18% (707 / 3_687)', 'remaining time': '26:35:33', 'throughput': '8317.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:40:54,847 >> {'loss': 0.3183, 'grad_norm': 9.922392845153809, 'learning_rate': 9.513736047011653e-07, 'epoch': 0.1920260374288039, 'num_input_tokens_seen': 1484783616, 'completed': '19.20% (708 / 3_687)', 'remaining time': '26:34:54', 'throughput': '8604.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:41:24,300 >> {'loss': 0.6533, 'grad_norm': 15.374136924743652, 'learning_rate': 9.511909150672721e-07, 'epoch': 0.19229726064551125, 'num_input_tokens_seen': 1486880768, 'completed': '19.23% (709 / 3_687)', 'remaining time': '26:34:11', 'throughput': '8900.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:41:53,905 >> {'loss': 0.5225, 'grad_norm': 16.63404083251953, 'learning_rate': 9.510079025701967e-07, 'epoch': 0.19256848386221861, 'num_input_tokens_seen': 1488977920, 'completed': '19.26% (710 / 3_687)', 'remaining time': '26:33:28', 'throughput': '8854.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:42:26,405 >> {'loss': 0.3678, 'grad_norm': 13.690201759338379, 'learning_rate': 9.508245673572209e-07, 'epoch': 0.19283970707892595, 'num_input_tokens_seen': 1491075072, 'completed': '19.28% (711 / 3_687)', 'remaining time': '26:32:57', 'throughput': '8066.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:42:57,344 >> {'loss': 0.352, 'grad_norm': 12.991211891174316, 'learning_rate': 9.50640909575886e-07, 'epoch': 0.1931109302956333, 'num_input_tokens_seen': 1493172224, 'completed': '19.31% (712 / 3_687)', 'remaining time': '26:32:20', 'throughput': '8472.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:43:27,653 >> {'loss': 0.7666, 'grad_norm': 18.690780639648438, 'learning_rate': 9.504569293739923e-07, 'epoch': 0.19338215351234064, 'num_input_tokens_seen': 1495269376, 'completed': '19.34% (713 / 3_687)', 'remaining time': '26:31:41', 'throughput': '8649.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:43:57,470 >> {'loss': 0.5814, 'grad_norm': 15.247132301330566, 'learning_rate': 9.502726268996005e-07, 'epoch': 0.193653376729048, 'num_input_tokens_seen': 1497366528, 'completed': '19.37% (714 / 3_687)', 'remaining time': '26:30:59', 'throughput': '8791.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:44:29,237 >> {'loss': 0.2609, 'grad_norm': 10.226189613342285, 'learning_rate': 9.5008800230103e-07, 'epoch': 0.19392459994575537, 'num_input_tokens_seen': 1499463680, 'completed': '19.39% (715 / 3_687)', 'remaining time': '26:30:26', 'throughput': '8252.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:44:58,526 >> {'loss': 0.6701, 'grad_norm': 19.02218246459961, 'learning_rate': 9.499030557268599e-07, 'epoch': 0.1941958231624627, 'num_input_tokens_seen': 1501560832, 'completed': '19.42% (716 / 3_687)', 'remaining time': '26:29:42', 'throughput': '8950.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:45:31,949 >> {'loss': 0.4341, 'grad_norm': 14.307744979858398, 'learning_rate': 9.497177873259279e-07, 'epoch': 0.19446704637917006, 'num_input_tokens_seen': 1503657984, 'completed': '19.45% (717 / 3_687)', 'remaining time': '26:29:15', 'throughput': '7843.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:46:03,559 >> {'loss': 0.3923, 'grad_norm': 15.487143516540527, 'learning_rate': 9.495321972473311e-07, 'epoch': 0.1947382695958774, 'num_input_tokens_seen': 1505755136, 'completed': '19.47% (718 / 3_687)', 'remaining time': '26:28:41', 'throughput': '8293.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:46:36,080 >> {'loss': 0.4535, 'grad_norm': 11.34209156036377, 'learning_rate': 9.493462856404251e-07, 'epoch': 0.19500949281258476, 'num_input_tokens_seen': 1507852288, 'completed': '19.50% (719 / 3_687)', 'remaining time': '26:28:11', 'throughput': '8060.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:47:05,168 >> {'loss': 0.4414, 'grad_norm': 13.599776268005371, 'learning_rate': 9.491600526548247e-07, 'epoch': 0.19528071602929212, 'num_input_tokens_seen': 1509949440, 'completed': '19.53% (720 / 3_687)', 'remaining time': '26:27:26', 'throughput': '9012.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:47:34,726 >> {'loss': 0.4073, 'grad_norm': 12.897449493408203, 'learning_rate': 9.489734984404033e-07, 'epoch': 0.19555193924599945, 'num_input_tokens_seen': 1512046592, 'completed': '19.56% (721 / 3_687)', 'remaining time': '26:26:43', 'throughput': '8868.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:48:04,129 >> {'loss': 0.5141, 'grad_norm': 16.54181480407715, 'learning_rate': 9.487866231472922e-07, 'epoch': 0.1958231624627068, 'num_input_tokens_seen': 1514143744, 'completed': '19.58% (722 / 3_687)', 'remaining time': '26:26:00', 'throughput': '8915.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:48:34,309 >> {'loss': 0.2837, 'grad_norm': 9.380181312561035, 'learning_rate': 9.48599426925882e-07, 'epoch': 0.19609438567941415, 'num_input_tokens_seen': 1516240896, 'completed': '19.61% (723 / 3_687)', 'remaining time': '26:25:20', 'throughput': '8686.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:49:06,625 >> {'loss': 0.6393, 'grad_norm': 17.064441680908203, 'learning_rate': 9.484119099268206e-07, 'epoch': 0.1963656088961215, 'num_input_tokens_seen': 1518338048, 'completed': '19.64% (724 / 3_687)', 'remaining time': '26:24:49', 'throughput': '8111.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:49:38,783 >> {'loss': 0.3783, 'grad_norm': 12.202215194702148, 'learning_rate': 9.482240723010148e-07, 'epoch': 0.19663683211282887, 'num_input_tokens_seen': 1520435200, 'completed': '19.66% (725 / 3_687)', 'remaining time': '26:24:17', 'throughput': '8151.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:50:07,725 >> {'loss': 0.7739, 'grad_norm': 20.838542938232422, 'learning_rate': 9.480359141996295e-07, 'epoch': 0.1969080553295362, 'num_input_tokens_seen': 1522532352, 'completed': '19.69% (726 / 3_687)', 'remaining time': '26:23:32', 'throughput': '9057.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:50:38,450 >> {'loss': 0.4288, 'grad_norm': 13.571410179138184, 'learning_rate': 9.478474357740864e-07, 'epoch': 0.19717927854624356, 'num_input_tokens_seen': 1524629504, 'completed': '19.72% (727 / 3_687)', 'remaining time': '26:22:55', 'throughput': '8531.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:51:12,572 >> {'loss': 0.4699, 'grad_norm': 13.46505355834961, 'learning_rate': 9.476586371760665e-07, 'epoch': 0.1974505017629509, 'num_input_tokens_seen': 1526726656, 'completed': '19.75% (728 / 3_687)', 'remaining time': '26:22:31', 'throughput': '7682.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:51:43,068 >> {'loss': 0.782, 'grad_norm': 16.68222427368164, 'learning_rate': 9.474695185575072e-07, 'epoch': 0.19772172497965826, 'num_input_tokens_seen': 1528823808, 'completed': '19.77% (729 / 3_687)', 'remaining time': '26:21:52', 'throughput': '8596.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:52:13,506 >> {'loss': 0.3531, 'grad_norm': 11.398141860961914, 'learning_rate': 9.472800800706044e-07, 'epoch': 0.19799294819636562, 'num_input_tokens_seen': 1530920960, 'completed': '19.80% (730 / 3_687)', 'remaining time': '26:21:14', 'throughput': '8612.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:52:41,540 >> {'loss': 0.8683, 'grad_norm': 17.724573135375977, 'learning_rate': 9.470903218678108e-07, 'epoch': 0.19826417141307295, 'num_input_tokens_seen': 1533018112, 'completed': '19.83% (731 / 3_687)', 'remaining time': '26:20:25', 'throughput': '9350.89', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:53:11,194 >> {'loss': 0.8129, 'grad_norm': 20.757295608520508, 'learning_rate': 9.469002441018366e-07, 'epoch': 0.19853539462978032, 'num_input_tokens_seen': 1535115264, 'completed': '19.85% (732 / 3_687)', 'remaining time': '26:19:43', 'throughput': '8840.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:53:42,997 >> {'loss': 0.5387, 'grad_norm': 16.623191833496094, 'learning_rate': 9.467098469256488e-07, 'epoch': 0.19880661784648765, 'num_input_tokens_seen': 1537212416, 'completed': '19.88% (733 / 3_687)', 'remaining time': '26:19:10', 'throughput': '8242.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:54:13,610 >> {'loss': 0.3834, 'grad_norm': 11.821105003356934, 'learning_rate': 9.465191304924725e-07, 'epoch': 0.199077841063195, 'num_input_tokens_seen': 1539309568, 'completed': '19.91% (734 / 3_687)', 'remaining time': '26:18:32', 'throughput': '8563.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:54:46,442 >> {'loss': 0.5618, 'grad_norm': 16.763580322265625, 'learning_rate': 9.463280949557885e-07, 'epoch': 0.19934906427990237, 'num_input_tokens_seen': 1541406720, 'completed': '19.93% (735 / 3_687)', 'remaining time': '26:18:03', 'throughput': '7984.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:55:18,227 >> {'loss': 0.5418, 'grad_norm': 14.250548362731934, 'learning_rate': 9.46136740469335e-07, 'epoch': 0.1996202874966097, 'num_input_tokens_seen': 1543503872, 'completed': '19.96% (736 / 3_687)', 'remaining time': '26:17:30', 'throughput': '8247.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:55:48,783 >> {'loss': 0.4692, 'grad_norm': 22.913976669311523, 'learning_rate': 9.45945067187107e-07, 'epoch': 0.19989151071331707, 'num_input_tokens_seen': 1545601024, 'completed': '19.99% (737 / 3_687)', 'remaining time': '26:16:52', 'throughput': '8579.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:56:18,712 >> {'loss': 0.522, 'grad_norm': 15.227788925170898, 'learning_rate': 9.457530752633557e-07, 'epoch': 0.2001627339300244, 'num_input_tokens_seen': 1547698176, 'completed': '20.02% (738 / 3_687)', 'remaining time': '26:16:11', 'throughput': '8758.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:56:52,852 >> {'loss': 0.2608, 'grad_norm': 12.005767822265625, 'learning_rate': 9.455607648525889e-07, 'epoch': 0.20043395714673176, 'num_input_tokens_seen': 1549795328, 'completed': '20.04% (739 / 3_687)', 'remaining time': '26:15:47', 'throughput': '7678.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:57:25,069 >> {'loss': 0.3001, 'grad_norm': 11.50250244140625, 'learning_rate': 9.45368136109571e-07, 'epoch': 0.20070518036343912, 'num_input_tokens_seen': 1551892480, 'completed': '20.07% (740 / 3_687)', 'remaining time': '26:15:16', 'throughput': '8136.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:57:52,152 >> {'loss': 0.6888, 'grad_norm': 19.086029052734375, 'learning_rate': 9.451751891893217e-07, 'epoch': 0.20097640358014646, 'num_input_tokens_seen': 1553989632, 'completed': '20.10% (741 / 3_687)', 'remaining time': '26:14:24', 'throughput': '9679.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:58:22,942 >> {'loss': 0.6411, 'grad_norm': 14.965021133422852, 'learning_rate': 9.449819242471179e-07, 'epoch': 0.20124762679685382, 'num_input_tokens_seen': 1556086784, 'completed': '20.12% (742 / 3_687)', 'remaining time': '26:13:47', 'throughput': '8513.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:58:52,802 >> {'loss': 0.4511, 'grad_norm': 12.905089378356934, 'learning_rate': 9.447883414384916e-07, 'epoch': 0.20151885001356115, 'num_input_tokens_seen': 1558183936, 'completed': '20.15% (743 / 3_687)', 'remaining time': '26:13:06', 'throughput': '8779.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:59:24,102 >> {'loss': 0.3277, 'grad_norm': 15.355229377746582, 'learning_rate': 9.445944409192308e-07, 'epoch': 0.2017900732302685, 'num_input_tokens_seen': 1560281088, 'completed': '20.18% (744 / 3_687)', 'remaining time': '26:12:31', 'throughput': '8375.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 17:59:55,135 >> {'loss': 0.3726, 'grad_norm': 12.57699203491211, 'learning_rate': 9.444002228453796e-07, 'epoch': 0.20206129644697587, 'num_input_tokens_seen': 1562378240, 'completed': '20.21% (745 / 3_687)', 'remaining time': '26:11:55', 'throughput': '8447.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:00:25,048 >> {'loss': 0.7487, 'grad_norm': 17.249740600585938, 'learning_rate': 9.442056873732369e-07, 'epoch': 0.2023325196636832, 'num_input_tokens_seen': 1564475392, 'completed': '20.23% (746 / 3_687)', 'remaining time': '26:11:14', 'throughput': '8763.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:00:57,903 >> {'loss': 0.6115, 'grad_norm': 19.854642868041992, 'learning_rate': 9.440108346593579e-07, 'epoch': 0.20260374288039057, 'num_input_tokens_seen': 1566572544, 'completed': '20.26% (747 / 3_687)', 'remaining time': '26:10:45', 'throughput': '7978.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:01:26,613 >> {'loss': 0.8311, 'grad_norm': 21.570199966430664, 'learning_rate': 9.438156648605521e-07, 'epoch': 0.2028749660970979, 'num_input_tokens_seen': 1568669696, 'completed': '20.29% (748 / 3_687)', 'remaining time': '26:10:00', 'throughput': '9130.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:01:56,950 >> {'loss': 0.4656, 'grad_norm': 14.75474739074707, 'learning_rate': 9.436201781338852e-07, 'epoch': 0.20314618931380526, 'num_input_tokens_seen': 1570766848, 'completed': '20.31% (749 / 3_687)', 'remaining time': '26:09:21', 'throughput': '8640.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:02:29,656 >> {'loss': 0.4637, 'grad_norm': 11.912012100219727, 'learning_rate': 9.434243746366771e-07, 'epoch': 0.20341741253051263, 'num_input_tokens_seen': 1572864000, 'completed': '20.34% (750 / 3_687)', 'remaining time': '26:08:52', 'throughput': '8015.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:02:59,207 >> {'loss': 0.5026, 'grad_norm': 18.71771240234375, 'learning_rate': 9.432282545265034e-07, 'epoch': 0.20368863574721996, 'num_input_tokens_seen': 1574961152, 'completed': '20.37% (751 / 3_687)', 'remaining time': '26:08:10', 'throughput': '8870.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:03:30,047 >> {'loss': 0.3925, 'grad_norm': 11.189618110656738, 'learning_rate': 9.430318179611938e-07, 'epoch': 0.20395985896392732, 'num_input_tokens_seen': 1577058304, 'completed': '20.40% (752 / 3_687)', 'remaining time': '26:07:33', 'throughput': '8500.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:04:00,833 >> {'loss': 0.6212, 'grad_norm': 15.765860557556152, 'learning_rate': 9.42835065098833e-07, 'epoch': 0.20423108218063465, 'num_input_tokens_seen': 1579155456, 'completed': '20.42% (753 / 3_687)', 'remaining time': '26:06:56', 'throughput': '8514.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:04:30,756 >> {'loss': 0.582, 'grad_norm': 19.61184310913086, 'learning_rate': 9.426379960977605e-07, 'epoch': 0.20450230539734202, 'num_input_tokens_seen': 1581252608, 'completed': '20.45% (754 / 3_687)', 'remaining time': '26:06:16', 'throughput': '8760.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:05:01,083 >> {'loss': 0.2286, 'grad_norm': 9.36821174621582, 'learning_rate': 9.424406111165697e-07, 'epoch': 0.20477352861404935, 'num_input_tokens_seen': 1583349760, 'completed': '20.48% (755 / 3_687)', 'remaining time': '26:05:37', 'throughput': '8643.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:05:30,594 >> {'loss': 1.0384, 'grad_norm': 25.251680374145508, 'learning_rate': 9.422429103141084e-07, 'epoch': 0.2050447518307567, 'num_input_tokens_seen': 1585446912, 'completed': '20.50% (756 / 3_687)', 'remaining time': '26:04:56', 'throughput': '8883.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:06:02,880 >> {'loss': 0.3857, 'grad_norm': 14.59071159362793, 'learning_rate': 9.42044893849479e-07, 'epoch': 0.20531597504746407, 'num_input_tokens_seen': 1587544064, 'completed': '20.53% (757 / 3_687)', 'remaining time': '26:04:25', 'throughput': '8119.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:06:35,687 >> {'loss': 0.6815, 'grad_norm': 16.961990356445312, 'learning_rate': 9.418465618820374e-07, 'epoch': 0.2055871982641714, 'num_input_tokens_seen': 1589641216, 'completed': '20.56% (758 / 3_687)', 'remaining time': '26:03:55', 'throughput': '7990.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:07:05,961 >> {'loss': 0.3136, 'grad_norm': 10.979452133178711, 'learning_rate': 9.416479145713936e-07, 'epoch': 0.20585842148087877, 'num_input_tokens_seen': 1591738368, 'completed': '20.59% (759 / 3_687)', 'remaining time': '26:03:17', 'throughput': '8659.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:07:36,627 >> {'loss': 0.5799, 'grad_norm': 14.783207893371582, 'learning_rate': 9.414489520774114e-07, 'epoch': 0.2061296446975861, 'num_input_tokens_seen': 1593835520, 'completed': '20.61% (760 / 3_687)', 'remaining time': '26:02:39', 'throughput': '8548.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:08:03,845 >> {'loss': 0.6763, 'grad_norm': 19.04448890686035, 'learning_rate': 9.412496745602084e-07, 'epoch': 0.20640086791429346, 'num_input_tokens_seen': 1595932672, 'completed': '20.64% (761 / 3_687)', 'remaining time': '26:01:49', 'throughput': '9631.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:08:37,639 >> {'loss': 0.6357, 'grad_norm': 24.7120418548584, 'learning_rate': 9.410500821801556e-07, 'epoch': 0.20667209113100082, 'num_input_tokens_seen': 1598029824, 'completed': '20.67% (762 / 3_687)', 'remaining time': '26:01:24', 'throughput': '7756.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:09:08,655 >> {'loss': 0.7047, 'grad_norm': 18.664295196533203, 'learning_rate': 9.408501750978769e-07, 'epoch': 0.20694331434770816, 'num_input_tokens_seen': 1600126976, 'completed': '20.69% (763 / 3_687)', 'remaining time': '26:00:48', 'throughput': '8451.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:09:40,150 >> {'loss': 0.3672, 'grad_norm': 12.567000389099121, 'learning_rate': 9.406499534742503e-07, 'epoch': 0.20721453756441552, 'num_input_tokens_seen': 1602224128, 'completed': '20.72% (764 / 3_687)', 'remaining time': '26:00:14', 'throughput': '8323.58', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:10:14,013 >> {'loss': 0.4399, 'grad_norm': 16.22792625427246, 'learning_rate': 9.404494174704068e-07, 'epoch': 0.20748576078112285, 'num_input_tokens_seen': 1604321280, 'completed': '20.75% (765 / 3_687)', 'remaining time': '25:59:49', 'throughput': '7741.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:10:46,547 >> {'loss': 0.3927, 'grad_norm': 11.791084289550781, 'learning_rate': 9.402485672477296e-07, 'epoch': 0.2077569839978302, 'num_input_tokens_seen': 1606418432, 'completed': '20.78% (766 / 3_687)', 'remaining time': '25:59:18', 'throughput': '8057.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:11:18,870 >> {'loss': 0.3697, 'grad_norm': 12.438189506530762, 'learning_rate': 9.400474029678555e-07, 'epoch': 0.20802820721453757, 'num_input_tokens_seen': 1608515584, 'completed': '20.80% (767 / 3_687)', 'remaining time': '25:58:48', 'throughput': '8110.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:11:49,409 >> {'loss': 0.5527, 'grad_norm': 14.400805473327637, 'learning_rate': 9.39845924792674e-07, 'epoch': 0.2082994304312449, 'num_input_tokens_seen': 1610612736, 'completed': '20.83% (768 / 3_687)', 'remaining time': '25:58:10', 'throughput': '8583.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:12:20,765 >> {'loss': 0.7031, 'grad_norm': 21.340023040771484, 'learning_rate': 9.396441328843268e-07, 'epoch': 0.20857065364795227, 'num_input_tokens_seen': 1612709888, 'completed': '20.86% (769 / 3_687)', 'remaining time': '25:57:35', 'throughput': '8360.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:12:51,654 >> {'loss': 0.6709, 'grad_norm': 18.95412254333496, 'learning_rate': 9.394420274052088e-07, 'epoch': 0.2088418768646596, 'num_input_tokens_seen': 1614807040, 'completed': '20.88% (770 / 3_687)', 'remaining time': '25:56:59', 'throughput': '8486.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:13:23,428 >> {'loss': 0.4891, 'grad_norm': 15.32834243774414, 'learning_rate': 9.392396085179662e-07, 'epoch': 0.20911310008136696, 'num_input_tokens_seen': 1616904192, 'completed': '20.91% (771 / 3_687)', 'remaining time': '25:56:26', 'throughput': '8250.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:13:55,062 >> {'loss': 0.5442, 'grad_norm': 14.922652244567871, 'learning_rate': 9.390368763854985e-07, 'epoch': 0.20938432329807433, 'num_input_tokens_seen': 1619001344, 'completed': '20.94% (772 / 3_687)', 'remaining time': '25:55:52', 'throughput': '8286.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:14:24,577 >> {'loss': 0.4359, 'grad_norm': 15.146223068237305, 'learning_rate': 9.388338311709566e-07, 'epoch': 0.20965554651478166, 'num_input_tokens_seen': 1621098496, 'completed': '20.97% (773 / 3_687)', 'remaining time': '25:55:11', 'throughput': '8881.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:14:54,850 >> {'loss': 0.7346, 'grad_norm': 18.45183753967285, 'learning_rate': 9.386304730377437e-07, 'epoch': 0.20992676973148902, 'num_input_tokens_seen': 1623195648, 'completed': '20.99% (774 / 3_687)', 'remaining time': '25:54:32', 'throughput': '8659.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:15:28,325 >> {'loss': 0.4655, 'grad_norm': 15.511632919311523, 'learning_rate': 9.384268021495145e-07, 'epoch': 0.21019799294819635, 'num_input_tokens_seen': 1625292800, 'completed': '21.02% (775 / 3_687)', 'remaining time': '25:54:06', 'throughput': '7831.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:15:57,647 >> {'loss': 0.4528, 'grad_norm': 12.674488067626953, 'learning_rate': 9.382228186701756e-07, 'epoch': 0.21046921616490372, 'num_input_tokens_seen': 1627389952, 'completed': '21.05% (776 / 3_687)', 'remaining time': '25:53:24', 'throughput': '8940.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:16:31,307 >> {'loss': 0.4335, 'grad_norm': 17.0650634765625, 'learning_rate': 9.380185227638854e-07, 'epoch': 0.21074043938161108, 'num_input_tokens_seen': 1629487104, 'completed': '21.07% (777 / 3_687)', 'remaining time': '25:52:58', 'throughput': '7787.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:17:03,554 >> {'loss': 0.4389, 'grad_norm': 13.060535430908203, 'learning_rate': 9.378139145950532e-07, 'epoch': 0.2110116625983184, 'num_input_tokens_seen': 1631584256, 'completed': '21.10% (778 / 3_687)', 'remaining time': '25:52:27', 'throughput': '8129.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:17:36,082 >> {'loss': 0.326, 'grad_norm': 11.724872589111328, 'learning_rate': 9.376089943283398e-07, 'epoch': 0.21128288581502577, 'num_input_tokens_seen': 1633681408, 'completed': '21.13% (779 / 3_687)', 'remaining time': '25:51:56', 'throughput': '8058.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:18:07,432 >> {'loss': 0.579, 'grad_norm': 20.703407287597656, 'learning_rate': 9.374037621286574e-07, 'epoch': 0.2115541090317331, 'num_input_tokens_seen': 1635778560, 'completed': '21.16% (780 / 3_687)', 'remaining time': '25:51:22', 'throughput': '8361.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:18:39,148 >> {'loss': 0.4883, 'grad_norm': 13.414408683776855, 'learning_rate': 9.371982181611692e-07, 'epoch': 0.21182533224844047, 'num_input_tokens_seen': 1637875712, 'completed': '21.18% (781 / 3_687)', 'remaining time': '25:50:49', 'throughput': '8265.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:19:09,919 >> {'loss': 0.495, 'grad_norm': 15.019268989562988, 'learning_rate': 9.369923625912888e-07, 'epoch': 0.21209655546514783, 'num_input_tokens_seen': 1639972864, 'completed': '21.21% (782 / 3_687)', 'remaining time': '25:50:12', 'throughput': '8519.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:19:40,871 >> {'loss': 0.3606, 'grad_norm': 11.590291976928711, 'learning_rate': 9.367861955846813e-07, 'epoch': 0.21236777868185516, 'num_input_tokens_seen': 1642070016, 'completed': '21.24% (783 / 3_687)', 'remaining time': '25:49:36', 'throughput': '8469.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:20:12,618 >> {'loss': 0.4129, 'grad_norm': 13.894659996032715, 'learning_rate': 9.365797173072619e-07, 'epoch': 0.21263900189856252, 'num_input_tokens_seen': 1644167168, 'completed': '21.26% (784 / 3_687)', 'remaining time': '25:49:03', 'throughput': '8257.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:20:40,580 >> {'loss': 0.634, 'grad_norm': 17.303585052490234, 'learning_rate': 9.363729279251965e-07, 'epoch': 0.21291022511526986, 'num_input_tokens_seen': 1646264320, 'completed': '21.29% (785 / 3_687)', 'remaining time': '25:48:16', 'throughput': '9374.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:21:11,289 >> {'loss': 0.6141, 'grad_norm': 15.712759017944336, 'learning_rate': 9.361658276049012e-07, 'epoch': 0.21318144833197722, 'num_input_tokens_seen': 1648361472, 'completed': '21.32% (786 / 3_687)', 'remaining time': '25:47:39', 'throughput': '8536.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:21:45,591 >> {'loss': 0.3697, 'grad_norm': 12.570011138916016, 'learning_rate': 9.359584165130426e-07, 'epoch': 0.21345267154868458, 'num_input_tokens_seen': 1650458624, 'completed': '21.35% (787 / 3_687)', 'remaining time': '25:47:16', 'throughput': '7642.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:22:14,363 >> {'loss': 0.6994, 'grad_norm': 17.42540740966797, 'learning_rate': 9.357506948165372e-07, 'epoch': 0.21372389476539191, 'num_input_tokens_seen': 1652555776, 'completed': '21.37% (788 / 3_687)', 'remaining time': '25:46:32', 'throughput': '9111.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:22:43,829 >> {'loss': 0.3642, 'grad_norm': 13.891875267028809, 'learning_rate': 9.355426626825516e-07, 'epoch': 0.21399511798209928, 'num_input_tokens_seen': 1654652928, 'completed': '21.40% (789 / 3_687)', 'remaining time': '25:45:50', 'throughput': '8896.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:23:14,076 >> {'loss': 0.4218, 'grad_norm': 14.662867546081543, 'learning_rate': 9.353343202785019e-07, 'epoch': 0.2142663411988066, 'num_input_tokens_seen': 1656750080, 'completed': '21.43% (790 / 3_687)', 'remaining time': '25:45:12', 'throughput': '8666.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:23:46,287 >> {'loss': 0.5427, 'grad_norm': 16.362796783447266, 'learning_rate': 9.351256677720542e-07, 'epoch': 0.21453756441551397, 'num_input_tokens_seen': 1658847232, 'completed': '21.45% (791 / 3_687)', 'remaining time': '25:44:41', 'throughput': '8138.44', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:24:15,696 >> {'loss': 0.6179, 'grad_norm': 16.475540161132812, 'learning_rate': 9.349167053311245e-07, 'epoch': 0.21480878763222133, 'num_input_tokens_seen': 1660944384, 'completed': '21.48% (792 / 3_687)', 'remaining time': '25:43:59', 'throughput': '8913.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:24:46,995 >> {'loss': 0.4276, 'grad_norm': 11.967279434204102, 'learning_rate': 9.347074331238774e-07, 'epoch': 0.21508001084892867, 'num_input_tokens_seen': 1663041536, 'completed': '21.51% (793 / 3_687)', 'remaining time': '25:43:25', 'throughput': '8375.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:25:17,196 >> {'loss': 0.2828, 'grad_norm': 10.945496559143066, 'learning_rate': 9.344978513187271e-07, 'epoch': 0.21535123406563603, 'num_input_tokens_seen': 1665138688, 'completed': '21.54% (794 / 3_687)', 'remaining time': '25:42:46', 'throughput': '8679.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:25:48,702 >> {'loss': 0.9658, 'grad_norm': 21.34172821044922, 'learning_rate': 9.342879600843376e-07, 'epoch': 0.21562245728234336, 'num_input_tokens_seen': 1667235840, 'completed': '21.56% (795 / 3_687)', 'remaining time': '25:42:12', 'throughput': '8320.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:26:20,139 >> {'loss': 0.4159, 'grad_norm': 13.481067657470703, 'learning_rate': 9.34077759589621e-07, 'epoch': 0.21589368049905072, 'num_input_tokens_seen': 1669332992, 'completed': '21.59% (796 / 3_687)', 'remaining time': '25:41:38', 'throughput': '8338.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:26:49,974 >> {'loss': 0.8269, 'grad_norm': 21.213205337524414, 'learning_rate': 9.338672500037387e-07, 'epoch': 0.21616490371575806, 'num_input_tokens_seen': 1671430144, 'completed': '21.62% (797 / 3_687)', 'remaining time': '25:40:59', 'throughput': '8786.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:27:19,525 >> {'loss': 0.4823, 'grad_norm': 12.533496856689453, 'learning_rate': 9.336564314961008e-07, 'epoch': 0.21643612693246542, 'num_input_tokens_seen': 1673527296, 'completed': '21.64% (798 / 3_687)', 'remaining time': '25:40:18', 'throughput': '8870.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:27:49,952 >> {'loss': 0.3558, 'grad_norm': 13.096843719482422, 'learning_rate': 9.334453042363661e-07, 'epoch': 0.21670735014917278, 'num_input_tokens_seen': 1675624448, 'completed': '21.67% (799 / 3_687)', 'remaining time': '25:39:40', 'throughput': '8615.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:28:20,772 >> {'loss': 0.8749, 'grad_norm': 21.356908798217773, 'learning_rate': 9.332338683944415e-07, 'epoch': 0.2169785733658801, 'num_input_tokens_seen': 1677721600, 'completed': '21.70% (800 / 3_687)', 'remaining time': '25:39:04', 'throughput': '8505.74', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2025-01-01 18:28:45,057 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-800 +[INFO|configuration_utils.py:472] 2025-01-01 18:28:45,060 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-800/config.json +[INFO|configuration_utils.py:807] 2025-01-01 18:28:45,061 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-800/generation_config.json +[INFO|modeling_utils.py:2807] 2025-01-01 18:29:40,626 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-800/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2025-01-01 18:29:40,630 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-800/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2025-01-01 18:29:40,631 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-800/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2025-01-01 18:33:26,349 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 1600, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2025-01-01 18:33:59,308 >> {'loss': 0.3332, 'grad_norm': 11.876173973083496, 'learning_rate': 9.33022124140483e-07, 'epoch': 0.21724979658258747, 'num_input_tokens_seen': 1679818752, 'completed': '21.72% (801 / 3_687)', 'remaining time': '25:56:56', 'throughput': '774.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:34:30,748 >> {'loss': 0.3932, 'grad_norm': 10.453437805175781, 'learning_rate': 9.32810071644894e-07, 'epoch': 0.2175210197992948, 'num_input_tokens_seen': 1681915904, 'completed': '21.75% (802 / 3_687)', 'remaining time': '25:56:21', 'throughput': '8337.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:35:01,659 >> {'loss': 0.323, 'grad_norm': 10.991945266723633, 'learning_rate': 9.325977110783263e-07, 'epoch': 0.21779224301600217, 'num_input_tokens_seen': 1684013056, 'completed': '21.78% (803 / 3_687)', 'remaining time': '25:55:43', 'throughput': '8480.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:35:29,192 >> {'loss': 0.5063, 'grad_norm': 15.58512020111084, 'learning_rate': 9.323850426116797e-07, 'epoch': 0.21806346623270953, 'num_input_tokens_seen': 1686110208, 'completed': '21.81% (804 / 3_687)', 'remaining time': '25:54:53', 'throughput': '9521.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:36:01,894 >> {'loss': 0.5303, 'grad_norm': 18.722299575805664, 'learning_rate': 9.321720664161017e-07, 'epoch': 0.21833468944941686, 'num_input_tokens_seen': 1688207360, 'completed': '21.83% (805 / 3_687)', 'remaining time': '25:54:22', 'throughput': '8015.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:36:31,380 >> {'loss': 0.4814, 'grad_norm': 17.243942260742188, 'learning_rate': 9.319587826629872e-07, 'epoch': 0.21860591266612422, 'num_input_tokens_seen': 1690304512, 'completed': '21.86% (806 / 3_687)', 'remaining time': '25:53:40', 'throughput': '8890.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:37:04,184 >> {'loss': 0.6408, 'grad_norm': 14.880188941955566, 'learning_rate': 9.317451915239792e-07, 'epoch': 0.21887713588283156, 'num_input_tokens_seen': 1692401664, 'completed': '21.89% (807 / 3_687)', 'remaining time': '25:53:09', 'throughput': '7991.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:37:37,603 >> {'loss': 0.5187, 'grad_norm': 14.305965423583984, 'learning_rate': 9.315312931709674e-07, 'epoch': 0.21914835909953892, 'num_input_tokens_seen': 1694498816, 'completed': '21.91% (808 / 3_687)', 'remaining time': '25:52:40', 'throughput': '7844.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:38:12,306 >> {'loss': 0.6711, 'grad_norm': 18.705963134765625, 'learning_rate': 9.313170877760892e-07, 'epoch': 0.21941958231624628, 'num_input_tokens_seen': 1696595968, 'completed': '21.94% (809 / 3_687)', 'remaining time': '25:52:16', 'throughput': '7553.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:38:43,415 >> {'loss': 0.4127, 'grad_norm': 14.972651481628418, 'learning_rate': 9.311025755117291e-07, 'epoch': 0.21969080553295361, 'num_input_tokens_seen': 1698693120, 'completed': '21.97% (810 / 3_687)', 'remaining time': '25:51:39', 'throughput': '8426.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:39:15,831 >> {'loss': 0.7297, 'grad_norm': 35.52400207519531, 'learning_rate': 9.308877565505181e-07, 'epoch': 0.21996202874966098, 'num_input_tokens_seen': 1700790272, 'completed': '22.00% (811 / 3_687)', 'remaining time': '25:51:07', 'throughput': '8086.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:39:47,894 >> {'loss': 0.3399, 'grad_norm': 11.86293888092041, 'learning_rate': 9.306726310653346e-07, 'epoch': 0.2202332519663683, 'num_input_tokens_seen': 1702887424, 'completed': '22.02% (812 / 3_687)', 'remaining time': '25:50:34', 'throughput': '8175.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:40:19,325 >> {'loss': 0.574, 'grad_norm': 15.077362060546875, 'learning_rate': 9.304571992293032e-07, 'epoch': 0.22050447518307567, 'num_input_tokens_seen': 1704984576, 'completed': '22.05% (813 / 3_687)', 'remaining time': '25:49:58', 'throughput': '8340.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:40:50,408 >> {'loss': 0.935, 'grad_norm': 24.58783531188965, 'learning_rate': 9.302414612157954e-07, 'epoch': 0.22077569839978303, 'num_input_tokens_seen': 1707081728, 'completed': '22.08% (814 / 3_687)', 'remaining time': '25:49:21', 'throughput': '8433.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:41:20,621 >> {'loss': 0.5214, 'grad_norm': 18.85881996154785, 'learning_rate': 9.300254171984289e-07, 'epoch': 0.22104692161649037, 'num_input_tokens_seen': 1709178880, 'completed': '22.10% (815 / 3_687)', 'remaining time': '25:48:41', 'throughput': '8676.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:41:54,017 >> {'loss': 0.6932, 'grad_norm': 23.3087158203125, 'learning_rate': 9.298090673510677e-07, 'epoch': 0.22131814483319773, 'num_input_tokens_seen': 1711276032, 'completed': '22.13% (816 / 3_687)', 'remaining time': '25:48:13', 'throughput': '7849.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:42:27,078 >> {'loss': 0.5998, 'grad_norm': 15.919090270996094, 'learning_rate': 9.295924118478218e-07, 'epoch': 0.22158936804990506, 'num_input_tokens_seen': 1713373184, 'completed': '22.16% (817 / 3_687)', 'remaining time': '25:47:43', 'throughput': '7929.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:42:55,240 >> {'loss': 0.7676, 'grad_norm': 19.33287239074707, 'learning_rate': 9.293754508630473e-07, 'epoch': 0.22186059126661242, 'num_input_tokens_seen': 1715470336, 'completed': '22.19% (818 / 3_687)', 'remaining time': '25:46:56', 'throughput': '9308.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:43:24,700 >> {'loss': 0.4777, 'grad_norm': 13.031168937683105, 'learning_rate': 9.291581845713466e-07, 'epoch': 0.22213181448331978, 'num_input_tokens_seen': 1717567488, 'completed': '22.21% (819 / 3_687)', 'remaining time': '25:46:13', 'throughput': '8898.16', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:43:57,232 >> {'loss': 0.4002, 'grad_norm': 13.561074256896973, 'learning_rate': 9.289406131475665e-07, 'epoch': 0.22240303770002712, 'num_input_tokens_seen': 1719664640, 'completed': '22.24% (820 / 3_687)', 'remaining time': '25:45:42', 'throughput': '8058.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:44:27,210 >> {'loss': 0.6063, 'grad_norm': 16.580875396728516, 'learning_rate': 9.287227367668012e-07, 'epoch': 0.22267426091673448, 'num_input_tokens_seen': 1721761792, 'completed': '22.27% (821 / 3_687)', 'remaining time': '25:45:01', 'throughput': '8744.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:45:02,093 >> {'loss': 0.3497, 'grad_norm': 11.243143081665039, 'learning_rate': 9.285045556043885e-07, 'epoch': 0.2229454841334418, 'num_input_tokens_seen': 1723858944, 'completed': '22.29% (822 / 3_687)', 'remaining time': '25:44:38', 'throughput': '7514.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:45:29,341 >> {'loss': 0.7562, 'grad_norm': 16.73326873779297, 'learning_rate': 9.282860698359128e-07, 'epoch': 0.22321670735014917, 'num_input_tokens_seen': 1725956096, 'completed': '22.32% (823 / 3_687)', 'remaining time': '25:43:47', 'throughput': '9620.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:45:57,190 >> {'loss': 0.9825, 'grad_norm': 19.721914291381836, 'learning_rate': 9.280672796372029e-07, 'epoch': 0.22348793056685654, 'num_input_tokens_seen': 1728053248, 'completed': '22.35% (824 / 3_687)', 'remaining time': '25:42:59', 'throughput': '9414.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:46:26,897 >> {'loss': 0.7322, 'grad_norm': 18.111730575561523, 'learning_rate': 9.278481851843327e-07, 'epoch': 0.22375915378356387, 'num_input_tokens_seen': 1730150400, 'completed': '22.38% (825 / 3_687)', 'remaining time': '25:42:18', 'throughput': '8823.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:46:59,700 >> {'loss': 0.7254, 'grad_norm': 20.524099349975586, 'learning_rate': 9.276287866536215e-07, 'epoch': 0.22403037700027123, 'num_input_tokens_seen': 1732247552, 'completed': '22.40% (826 / 3_687)', 'remaining time': '25:41:47', 'throughput': '7991.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:47:30,423 >> {'loss': 0.3707, 'grad_norm': 13.87217903137207, 'learning_rate': 9.274090842216326e-07, 'epoch': 0.22430160021697856, 'num_input_tokens_seen': 1734344704, 'completed': '22.43% (827 / 3_687)', 'remaining time': '25:41:09', 'throughput': '8532.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:48:03,191 >> {'loss': 0.7017, 'grad_norm': 19.11530113220215, 'learning_rate': 9.271890780651741e-07, 'epoch': 0.22457282343368593, 'num_input_tokens_seen': 1736441856, 'completed': '22.46% (828 / 3_687)', 'remaining time': '25:40:39', 'throughput': '8000.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:48:34,676 >> {'loss': 0.2848, 'grad_norm': 9.669900894165039, 'learning_rate': 9.269687683612987e-07, 'epoch': 0.2248440466503933, 'num_input_tokens_seen': 1738539008, 'completed': '22.48% (829 / 3_687)', 'remaining time': '25:40:03', 'throughput': '8326.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:49:04,441 >> {'loss': 0.4288, 'grad_norm': 14.931427955627441, 'learning_rate': 9.267481552873033e-07, 'epoch': 0.22511526986710062, 'num_input_tokens_seen': 1740636160, 'completed': '22.51% (830 / 3_687)', 'remaining time': '25:39:22', 'throughput': '8807.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:49:35,405 >> {'loss': 0.5282, 'grad_norm': 16.54037094116211, 'learning_rate': 9.265272390207289e-07, 'epoch': 0.22538649308380798, 'num_input_tokens_seen': 1742733312, 'completed': '22.54% (831 / 3_687)', 'remaining time': '25:38:45', 'throughput': '8466.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:50:04,174 >> {'loss': 0.5665, 'grad_norm': 25.891746520996094, 'learning_rate': 9.263060197393603e-07, 'epoch': 0.22565771630051532, 'num_input_tokens_seen': 1744830464, 'completed': '22.57% (832 / 3_687)', 'remaining time': '25:38:01', 'throughput': '9112.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:50:31,859 >> {'loss': 0.8732, 'grad_norm': 18.841882705688477, 'learning_rate': 9.260844976212268e-07, 'epoch': 0.22592893951722268, 'num_input_tokens_seen': 1746927616, 'completed': '22.59% (833 / 3_687)', 'remaining time': '25:37:12', 'throughput': '9468.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:51:05,688 >> {'loss': 0.4302, 'grad_norm': 12.339317321777344, 'learning_rate': 9.258626728446004e-07, 'epoch': 0.22620016273393004, 'num_input_tokens_seen': 1749024768, 'completed': '22.62% (834 / 3_687)', 'remaining time': '25:36:45', 'throughput': '7748.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:51:33,658 >> {'loss': 0.7744, 'grad_norm': 18.42291831970215, 'learning_rate': 9.256405455879977e-07, 'epoch': 0.22647138595063737, 'num_input_tokens_seen': 1751121920, 'completed': '22.65% (835 / 3_687)', 'remaining time': '25:35:58', 'throughput': '9372.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:52:03,473 >> {'loss': 0.726, 'grad_norm': 22.923080444335938, 'learning_rate': 9.25418116030178e-07, 'epoch': 0.22674260916734473, 'num_input_tokens_seen': 1753219072, 'completed': '22.67% (836 / 3_687)', 'remaining time': '25:35:17', 'throughput': '8792.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:52:35,260 >> {'loss': 0.3722, 'grad_norm': 12.434412956237793, 'learning_rate': 9.251953843501443e-07, 'epoch': 0.22701383238405207, 'num_input_tokens_seen': 1755316224, 'completed': '22.70% (837 / 3_687)', 'remaining time': '25:34:43', 'throughput': '8247.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:53:06,178 >> {'loss': 0.5697, 'grad_norm': 18.6683292388916, 'learning_rate': 9.249723507271425e-07, 'epoch': 0.22728505560075943, 'num_input_tokens_seen': 1757413376, 'completed': '22.73% (838 / 3_687)', 'remaining time': '25:34:06', 'throughput': '8478.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:53:37,101 >> {'loss': 0.5072, 'grad_norm': 14.937889099121094, 'learning_rate': 9.247490153406617e-07, 'epoch': 0.22755627881746676, 'num_input_tokens_seen': 1759510528, 'completed': '22.76% (839 / 3_687)', 'remaining time': '25:33:29', 'throughput': '8477.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:54:09,195 >> {'loss': 0.3661, 'grad_norm': 12.447941780090332, 'learning_rate': 9.245253783704334e-07, 'epoch': 0.22782750203417412, 'num_input_tokens_seen': 1761607680, 'completed': '22.78% (840 / 3_687)', 'remaining time': '25:32:56', 'throughput': '8167.98', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:54:39,436 >> {'loss': 0.3809, 'grad_norm': 11.856642723083496, 'learning_rate': 9.243014399964324e-07, 'epoch': 0.22809872525088148, 'num_input_tokens_seen': 1763704832, 'completed': '22.81% (841 / 3_687)', 'remaining time': '25:32:17', 'throughput': '8668.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:55:11,102 >> {'loss': 0.3321, 'grad_norm': 10.702392578125, 'learning_rate': 9.240772003988758e-07, 'epoch': 0.22836994846758882, 'num_input_tokens_seen': 1765801984, 'completed': '22.84% (842 / 3_687)', 'remaining time': '25:31:42', 'throughput': '8278.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:55:42,035 >> {'loss': 0.5027, 'grad_norm': 13.526901245117188, 'learning_rate': 9.238526597582229e-07, 'epoch': 0.22864117168429618, 'num_input_tokens_seen': 1767899136, 'completed': '22.86% (843 / 3_687)', 'remaining time': '25:31:05', 'throughput': '8474.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:56:11,399 >> {'loss': 0.7197, 'grad_norm': 22.885881423950195, 'learning_rate': 9.236278182551758e-07, 'epoch': 0.2289123949010035, 'num_input_tokens_seen': 1769996288, 'completed': '22.89% (844 / 3_687)', 'remaining time': '25:30:23', 'throughput': '8927.34', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:56:39,138 >> {'loss': 0.5759, 'grad_norm': 20.40297508239746, 'learning_rate': 9.23402676070678e-07, 'epoch': 0.22918361811771087, 'num_input_tokens_seen': 1772093440, 'completed': '22.92% (845 / 3_687)', 'remaining time': '25:29:36', 'throughput': '9450.38', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:57:11,292 >> {'loss': 0.6163, 'grad_norm': 19.0419921875, 'learning_rate': 9.231772333859154e-07, 'epoch': 0.22945484133441824, 'num_input_tokens_seen': 1774190592, 'completed': '22.95% (846 / 3_687)', 'remaining time': '25:29:03', 'throughput': '8152.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:57:41,371 >> {'loss': 0.3334, 'grad_norm': 12.630133628845215, 'learning_rate': 9.22951490382316e-07, 'epoch': 0.22972606455112557, 'num_input_tokens_seen': 1776287744, 'completed': '22.97% (847 / 3_687)', 'remaining time': '25:28:23', 'throughput': '8715.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:58:12,463 >> {'loss': 0.3219, 'grad_norm': 11.405631065368652, 'learning_rate': 9.22725447241549e-07, 'epoch': 0.22999728776783293, 'num_input_tokens_seen': 1778384896, 'completed': '23.00% (848 / 3_687)', 'remaining time': '25:27:47', 'throughput': '8431.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:58:46,491 >> {'loss': 0.4205, 'grad_norm': 14.75662899017334, 'learning_rate': 9.224991041455252e-07, 'epoch': 0.23026851098454026, 'num_input_tokens_seen': 1780482048, 'completed': '23.03% (849 / 3_687)', 'remaining time': '25:27:20', 'throughput': '7703.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:59:16,974 >> {'loss': 0.6833, 'grad_norm': 16.446325302124023, 'learning_rate': 9.222724612763971e-07, 'epoch': 0.23053973420124763, 'num_input_tokens_seen': 1782579200, 'completed': '23.05% (850 / 3_687)', 'remaining time': '25:26:42', 'throughput': '8599.47', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 18:59:45,874 >> {'loss': 0.5999, 'grad_norm': 14.62348747253418, 'learning_rate': 9.220455188165582e-07, 'epoch': 0.230810957417955, 'num_input_tokens_seen': 1784676352, 'completed': '23.08% (851 / 3_687)', 'remaining time': '25:25:58', 'throughput': '9070.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:00:16,214 >> {'loss': 0.7337, 'grad_norm': 20.784975051879883, 'learning_rate': 9.218182769486433e-07, 'epoch': 0.23108218063466232, 'num_input_tokens_seen': 1786773504, 'completed': '23.11% (852 / 3_687)', 'remaining time': '25:25:20', 'throughput': '8640.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:00:44,107 >> {'loss': 0.6725, 'grad_norm': 16.6190128326416, 'learning_rate': 9.215907358555276e-07, 'epoch': 0.23135340385136968, 'num_input_tokens_seen': 1788870656, 'completed': '23.14% (853 / 3_687)', 'remaining time': '25:24:33', 'throughput': '9398.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:01:16,469 >> {'loss': 0.8521, 'grad_norm': 19.758882522583008, 'learning_rate': 9.213628957203277e-07, 'epoch': 0.23162462706807702, 'num_input_tokens_seen': 1790967808, 'completed': '23.16% (854 / 3_687)', 'remaining time': '25:24:01', 'throughput': '8100.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:01:47,281 >> {'loss': 0.7921, 'grad_norm': 21.532390594482422, 'learning_rate': 9.21134756726401e-07, 'epoch': 0.23189585028478438, 'num_input_tokens_seen': 1793064960, 'completed': '23.19% (855 / 3_687)', 'remaining time': '25:23:24', 'throughput': '8507.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:02:18,180 >> {'loss': 0.673, 'grad_norm': 19.4443302154541, 'learning_rate': 9.209063190573445e-07, 'epoch': 0.23216707350149174, 'num_input_tokens_seen': 1795162112, 'completed': '23.22% (856 / 3_687)', 'remaining time': '25:22:47', 'throughput': '8483.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:02:50,930 >> {'loss': 0.3932, 'grad_norm': 14.834383964538574, 'learning_rate': 9.206775828969967e-07, 'epoch': 0.23243829671819907, 'num_input_tokens_seen': 1797259264, 'completed': '23.24% (857 / 3_687)', 'remaining time': '25:22:16', 'throughput': '8004.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:03:22,854 >> {'loss': 0.37, 'grad_norm': 12.486455917358398, 'learning_rate': 9.204485484294355e-07, 'epoch': 0.23270951993490643, 'num_input_tokens_seen': 1799356416, 'completed': '23.27% (858 / 3_687)', 'remaining time': '25:21:43', 'throughput': '8211.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:03:52,965 >> {'loss': 0.6087, 'grad_norm': 15.540726661682129, 'learning_rate': 9.202192158389791e-07, 'epoch': 0.23298074315161377, 'num_input_tokens_seen': 1801453568, 'completed': '23.30% (859 / 3_687)', 'remaining time': '25:21:03', 'throughput': '8705.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:04:23,939 >> {'loss': 0.7186, 'grad_norm': 16.4538631439209, 'learning_rate': 9.199895853101856e-07, 'epoch': 0.23325196636832113, 'num_input_tokens_seen': 1803550720, 'completed': '23.33% (860 / 3_687)', 'remaining time': '25:20:27', 'throughput': '8463.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:04:52,665 >> {'loss': 0.7831, 'grad_norm': 18.737592697143555, 'learning_rate': 9.197596570278529e-07, 'epoch': 0.2335231895850285, 'num_input_tokens_seen': 1805647872, 'completed': '23.35% (861 / 3_687)', 'remaining time': '25:19:43', 'throughput': '9125.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:05:22,796 >> {'loss': 0.3793, 'grad_norm': 13.706647872924805, 'learning_rate': 9.19529431177019e-07, 'epoch': 0.23379441280173582, 'num_input_tokens_seen': 1807745024, 'completed': '23.38% (862 / 3_687)', 'remaining time': '25:19:04', 'throughput': '8700.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:05:52,156 >> {'loss': 0.1998, 'grad_norm': 11.725102424621582, 'learning_rate': 9.192989079429603e-07, 'epoch': 0.23406563601844319, 'num_input_tokens_seen': 1809842176, 'completed': '23.41% (863 / 3_687)', 'remaining time': '25:18:22', 'throughput': '8928.60', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:06:23,082 >> {'loss': 0.4601, 'grad_norm': 15.088400840759277, 'learning_rate': 9.190680875111934e-07, 'epoch': 0.23433685923515052, 'num_input_tokens_seen': 1811939328, 'completed': '23.43% (864 / 3_687)', 'remaining time': '25:17:45', 'throughput': '8476.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:06:56,739 >> {'loss': 0.4754, 'grad_norm': 14.075953483581543, 'learning_rate': 9.188369700674735e-07, 'epoch': 0.23460808245185788, 'num_input_tokens_seen': 1814036480, 'completed': '23.46% (865 / 3_687)', 'remaining time': '25:17:18', 'throughput': '7788.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:07:29,552 >> {'loss': 0.5167, 'grad_norm': 19.2708740234375, 'learning_rate': 9.186055557977957e-07, 'epoch': 0.23487930566856524, 'num_input_tokens_seen': 1816133632, 'completed': '23.49% (866 / 3_687)', 'remaining time': '25:16:47', 'throughput': '7989.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:07:58,558 >> {'loss': 0.7142, 'grad_norm': 18.750097274780273, 'learning_rate': 9.18373844888393e-07, 'epoch': 0.23515052888527258, 'num_input_tokens_seen': 1818230784, 'completed': '23.52% (867 / 3_687)', 'remaining time': '25:16:04', 'throughput': '9037.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:08:30,437 >> {'loss': 0.4226, 'grad_norm': 13.886237144470215, 'learning_rate': 9.181418375257374e-07, 'epoch': 0.23542175210197994, 'num_input_tokens_seen': 1820327936, 'completed': '23.54% (868 / 3_687)', 'remaining time': '25:15:31', 'throughput': '8223.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:08:58,688 >> {'loss': 0.6679, 'grad_norm': 18.76662254333496, 'learning_rate': 9.179095338965401e-07, 'epoch': 0.23569297531868727, 'num_input_tokens_seen': 1822425088, 'completed': '23.57% (869 / 3_687)', 'remaining time': '25:14:45', 'throughput': '9278.93', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:09:31,052 >> {'loss': 0.4632, 'grad_norm': 13.589776039123535, 'learning_rate': 9.176769341877497e-07, 'epoch': 0.23596419853539463, 'num_input_tokens_seen': 1824522240, 'completed': '23.60% (870 / 3_687)', 'remaining time': '25:14:14', 'throughput': '8099.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:10:00,796 >> {'loss': 0.5446, 'grad_norm': 16.83132553100586, 'learning_rate': 9.17444038586554e-07, 'epoch': 0.236235421752102, 'num_input_tokens_seen': 1826619392, 'completed': '23.62% (871 / 3_687)', 'remaining time': '25:13:33', 'throughput': '8813.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:10:31,603 >> {'loss': 0.6821, 'grad_norm': 26.304426193237305, 'learning_rate': 9.172108472803782e-07, 'epoch': 0.23650664496880933, 'num_input_tokens_seen': 1828716544, 'completed': '23.65% (872 / 3_687)', 'remaining time': '25:12:56', 'throughput': '8509.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:11:03,723 >> {'loss': 0.4784, 'grad_norm': 18.7716121673584, 'learning_rate': 9.16977360456886e-07, 'epoch': 0.2367778681855167, 'num_input_tokens_seen': 1830813696, 'completed': '23.68% (873 / 3_687)', 'remaining time': '25:12:24', 'throughput': '8161.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:11:36,260 >> {'loss': 0.3455, 'grad_norm': 11.739683151245117, 'learning_rate': 9.167435783039786e-07, 'epoch': 0.23704909140222402, 'num_input_tokens_seen': 1832910848, 'completed': '23.70% (874 / 3_687)', 'remaining time': '25:11:52', 'throughput': '8056.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:12:06,073 >> {'loss': 0.6126, 'grad_norm': 14.229680061340332, 'learning_rate': 9.165095010097949e-07, 'epoch': 0.23732031461893138, 'num_input_tokens_seen': 1835008000, 'completed': '23.73% (875 / 3_687)', 'remaining time': '25:11:12', 'throughput': '8793.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:12:37,087 >> {'loss': 0.6215, 'grad_norm': 15.295489311218262, 'learning_rate': 9.162751287627116e-07, 'epoch': 0.23759153783563874, 'num_input_tokens_seen': 1837105152, 'completed': '23.76% (876 / 3_687)', 'remaining time': '25:10:36', 'throughput': '8452.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:13:08,185 >> {'loss': 0.4853, 'grad_norm': 13.08266830444336, 'learning_rate': 9.160404617513424e-07, 'epoch': 0.23786276105234608, 'num_input_tokens_seen': 1839202304, 'completed': '23.79% (877 / 3_687)', 'remaining time': '25:10:00', 'throughput': '8429.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:13:36,779 >> {'loss': 0.7992, 'grad_norm': 18.74358558654785, 'learning_rate': 9.158055001645385e-07, 'epoch': 0.23813398426905344, 'num_input_tokens_seen': 1841299456, 'completed': '23.81% (878 / 3_687)', 'remaining time': '25:09:16', 'throughput': '9167.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:14:08,253 >> {'loss': 0.4748, 'grad_norm': 14.171740531921387, 'learning_rate': 9.155702441913881e-07, 'epoch': 0.23840520748576077, 'num_input_tokens_seen': 1843396608, 'completed': '23.84% (879 / 3_687)', 'remaining time': '25:08:42', 'throughput': '8328.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:14:38,976 >> {'loss': 0.5375, 'grad_norm': 15.582724571228027, 'learning_rate': 9.15334694021216e-07, 'epoch': 0.23867643070246813, 'num_input_tokens_seen': 1845493760, 'completed': '23.87% (880 / 3_687)', 'remaining time': '25:08:04', 'throughput': '8532.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:15:08,496 >> {'loss': 0.5576, 'grad_norm': 16.704227447509766, 'learning_rate': 9.150988498435843e-07, 'epoch': 0.23894765391917547, 'num_input_tokens_seen': 1847590912, 'completed': '23.89% (881 / 3_687)', 'remaining time': '25:07:24', 'throughput': '8880.18', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:15:41,773 >> {'loss': 0.287, 'grad_norm': 12.607769966125488, 'learning_rate': 9.148627118482912e-07, 'epoch': 0.23921887713588283, 'num_input_tokens_seen': 1849688064, 'completed': '23.92% (882 / 3_687)', 'remaining time': '25:06:55', 'throughput': '7877.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:16:10,169 >> {'loss': 0.752, 'grad_norm': 16.634292602539062, 'learning_rate': 9.146262802253717e-07, 'epoch': 0.2394901003525902, 'num_input_tokens_seen': 1851785216, 'completed': '23.95% (883 / 3_687)', 'remaining time': '25:06:10', 'throughput': '9231.79', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:16:41,675 >> {'loss': 0.3663, 'grad_norm': 13.200092315673828, 'learning_rate': 9.14389555165097e-07, 'epoch': 0.23976132356929752, 'num_input_tokens_seen': 1853882368, 'completed': '23.98% (884 / 3_687)', 'remaining time': '25:05:36', 'throughput': '8320.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:17:14,535 >> {'loss': 0.507, 'grad_norm': 15.857352256774902, 'learning_rate': 9.141525368579742e-07, 'epoch': 0.24003254678600489, 'num_input_tokens_seen': 1855979520, 'completed': '24.00% (885 / 3_687)', 'remaining time': '25:05:06', 'throughput': '7977.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:17:46,516 >> {'loss': 0.4418, 'grad_norm': 15.330340385437012, 'learning_rate': 9.139152254947469e-07, 'epoch': 0.24030377000271222, 'num_input_tokens_seen': 1858076672, 'completed': '24.03% (886 / 3_687)', 'remaining time': '25:04:33', 'throughput': '8196.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:18:19,923 >> {'loss': 0.4606, 'grad_norm': 12.231500625610352, 'learning_rate': 9.136776212663942e-07, 'epoch': 0.24057499321941958, 'num_input_tokens_seen': 1860173824, 'completed': '24.06% (887 / 3_687)', 'remaining time': '25:04:04', 'throughput': '7846.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:18:50,844 >> {'loss': 0.6261, 'grad_norm': 19.30156898498535, 'learning_rate': 9.134397243641307e-07, 'epoch': 0.24084621643612694, 'num_input_tokens_seen': 1862270976, 'completed': '24.08% (888 / 3_687)', 'remaining time': '25:03:28', 'throughput': '8478.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:19:24,075 >> {'loss': 0.4225, 'grad_norm': 23.425487518310547, 'learning_rate': 9.132015349794069e-07, 'epoch': 0.24111743965283428, 'num_input_tokens_seen': 1864368128, 'completed': '24.11% (889 / 3_687)', 'remaining time': '25:02:59', 'throughput': '7888.53', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:19:56,381 >> {'loss': 0.2508, 'grad_norm': 9.727029800415039, 'learning_rate': 9.129630533039086e-07, 'epoch': 0.24138866286954164, 'num_input_tokens_seen': 1866465280, 'completed': '24.14% (890 / 3_687)', 'remaining time': '25:02:27', 'throughput': '8114.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:20:29,790 >> {'loss': 0.7279, 'grad_norm': 20.487241744995117, 'learning_rate': 9.127242795295569e-07, 'epoch': 0.24165988608624897, 'num_input_tokens_seen': 1868562432, 'completed': '24.17% (891 / 3_687)', 'remaining time': '25:01:58', 'throughput': '7846.57', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:20:59,885 >> {'loss': 0.5699, 'grad_norm': 16.180301666259766, 'learning_rate': 9.124852138485076e-07, 'epoch': 0.24193110930295633, 'num_input_tokens_seen': 1870659584, 'completed': '24.19% (892 / 3_687)', 'remaining time': '25:01:19', 'throughput': '8710.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:21:31,720 >> {'loss': 0.856, 'grad_norm': 20.350139617919922, 'learning_rate': 9.12245856453152e-07, 'epoch': 0.2422023325196637, 'num_input_tokens_seen': 1872756736, 'completed': '24.22% (893 / 3_687)', 'remaining time': '25:00:46', 'throughput': '8234.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:22:04,504 >> {'loss': 0.6445, 'grad_norm': 18.013940811157227, 'learning_rate': 9.120062075361155e-07, 'epoch': 0.24247355573637103, 'num_input_tokens_seen': 1874853888, 'completed': '24.25% (894 / 3_687)', 'remaining time': '25:00:15', 'throughput': '7995.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:22:34,351 >> {'loss': 0.6282, 'grad_norm': 16.004846572875977, 'learning_rate': 9.117662672902584e-07, 'epoch': 0.2427447789530784, 'num_input_tokens_seen': 1876951040, 'completed': '24.27% (895 / 3_687)', 'remaining time': '24:59:36', 'throughput': '8782.88', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:23:06,012 >> {'loss': 0.3772, 'grad_norm': 14.721831321716309, 'learning_rate': 9.115260359086757e-07, 'epoch': 0.24301600216978572, 'num_input_tokens_seen': 1879048192, 'completed': '24.30% (896 / 3_687)', 'remaining time': '24:59:02', 'throughput': '8279.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:23:35,354 >> {'loss': 0.5968, 'grad_norm': 19.249744415283203, 'learning_rate': 9.112855135846964e-07, 'epoch': 0.24328722538649308, 'num_input_tokens_seen': 1881145344, 'completed': '24.33% (897 / 3_687)', 'remaining time': '24:58:20', 'throughput': '8934.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:24:05,956 >> {'loss': 0.4154, 'grad_norm': 12.756824493408203, 'learning_rate': 9.110447005118836e-07, 'epoch': 0.24355844860320044, 'num_input_tokens_seen': 1883242496, 'completed': '24.36% (898 / 3_687)', 'remaining time': '24:57:43', 'throughput': '8566.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:24:38,618 >> {'loss': 0.3552, 'grad_norm': 11.557883262634277, 'learning_rate': 9.108035968840348e-07, 'epoch': 0.24382967181990778, 'num_input_tokens_seen': 1885339648, 'completed': '24.38% (899 / 3_687)', 'remaining time': '24:57:12', 'throughput': '8026.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:25:10,857 >> {'loss': 0.6714, 'grad_norm': 19.659439086914062, 'learning_rate': 9.105622028951806e-07, 'epoch': 0.24410089503661514, 'num_input_tokens_seen': 1887436800, 'completed': '24.41% (900 / 3_687)', 'remaining time': '24:56:40', 'throughput': '8131.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:25:44,816 >> {'loss': 0.367, 'grad_norm': 15.646737098693848, 'learning_rate': 9.103205187395861e-07, 'epoch': 0.24437211825332247, 'num_input_tokens_seen': 1889533952, 'completed': '24.44% (901 / 3_687)', 'remaining time': '24:56:13', 'throughput': '7719.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:26:15,580 >> {'loss': 0.4032, 'grad_norm': 13.878365516662598, 'learning_rate': 9.100785446117493e-07, 'epoch': 0.24464334147002983, 'num_input_tokens_seen': 1891631104, 'completed': '24.46% (902 / 3_687)', 'remaining time': '24:55:36', 'throughput': '8521.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:26:44,245 >> {'loss': 0.5326, 'grad_norm': 16.992267608642578, 'learning_rate': 9.098362807064017e-07, 'epoch': 0.2449145646867372, 'num_input_tokens_seen': 1893728256, 'completed': '24.49% (903 / 3_687)', 'remaining time': '24:54:53', 'throughput': '9144.95', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:27:13,970 >> {'loss': 0.3813, 'grad_norm': 12.592440605163574, 'learning_rate': 9.095937272185083e-07, 'epoch': 0.24518578790344453, 'num_input_tokens_seen': 1895825408, 'completed': '24.52% (904 / 3_687)', 'remaining time': '24:54:13', 'throughput': '8818.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:27:45,084 >> {'loss': 0.449, 'grad_norm': 14.522198677062988, 'learning_rate': 9.093508843432667e-07, 'epoch': 0.2454570111201519, 'num_input_tokens_seen': 1897922560, 'completed': '24.55% (905 / 3_687)', 'remaining time': '24:53:38', 'throughput': '8425.27', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:28:15,872 >> {'loss': 0.6901, 'grad_norm': 19.913555145263672, 'learning_rate': 9.091077522761078e-07, 'epoch': 0.24572823433685922, 'num_input_tokens_seen': 1900019712, 'completed': '24.57% (906 / 3_687)', 'remaining time': '24:53:01', 'throughput': '8514.45', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:28:45,875 >> {'loss': 0.7029, 'grad_norm': 18.202465057373047, 'learning_rate': 9.088643312126948e-07, 'epoch': 0.2459994575535666, 'num_input_tokens_seen': 1902116864, 'completed': '24.60% (907 / 3_687)', 'remaining time': '24:52:22', 'throughput': '8737.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:29:18,151 >> {'loss': 0.4506, 'grad_norm': 13.385407447814941, 'learning_rate': 9.086206213489239e-07, 'epoch': 0.24627068077027395, 'num_input_tokens_seen': 1904214016, 'completed': '24.63% (908 / 3_687)', 'remaining time': '24:51:50', 'throughput': '8122.00', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:29:49,509 >> {'loss': 0.6725, 'grad_norm': 17.774667739868164, 'learning_rate': 9.083766228809234e-07, 'epoch': 0.24654190398698128, 'num_input_tokens_seen': 1906311168, 'completed': '24.65% (909 / 3_687)', 'remaining time': '24:51:15', 'throughput': '8359.66', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:30:20,015 >> {'loss': 0.4936, 'grad_norm': 14.268959999084473, 'learning_rate': 9.081323360050543e-07, 'epoch': 0.24681312720368864, 'num_input_tokens_seen': 1908408320, 'completed': '24.68% (910 / 3_687)', 'remaining time': '24:50:38', 'throughput': '8593.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:30:50,570 >> {'loss': 0.4626, 'grad_norm': 15.435829162597656, 'learning_rate': 9.078877609179088e-07, 'epoch': 0.24708435042039598, 'num_input_tokens_seen': 1910505472, 'completed': '24.71% (911 / 3_687)', 'remaining time': '24:50:01', 'throughput': '8579.33', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:31:20,695 >> {'loss': 0.337, 'grad_norm': 11.141834259033203, 'learning_rate': 9.076428978163121e-07, 'epoch': 0.24735557363710334, 'num_input_tokens_seen': 1912602624, 'completed': '24.74% (912 / 3_687)', 'remaining time': '24:49:22', 'throughput': '8702.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:31:48,447 >> {'loss': 0.993, 'grad_norm': 21.919893264770508, 'learning_rate': 9.073977468973206e-07, 'epoch': 0.2476267968538107, 'num_input_tokens_seen': 1914699776, 'completed': '24.76% (913 / 3_687)', 'remaining time': '24:48:37', 'throughput': '9445.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:32:17,805 >> {'loss': 0.3735, 'grad_norm': 15.084315299987793, 'learning_rate': 9.071523083582223e-07, 'epoch': 0.24789802007051803, 'num_input_tokens_seen': 1916796928, 'completed': '24.79% (914 / 3_687)', 'remaining time': '24:47:56', 'throughput': '8929.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:32:47,086 >> {'loss': 0.7275, 'grad_norm': 19.048635482788086, 'learning_rate': 9.06906582396537e-07, 'epoch': 0.2481692432872254, 'num_input_tokens_seen': 1918894080, 'completed': '24.82% (915 / 3_687)', 'remaining time': '24:47:15', 'throughput': '8952.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:33:19,466 >> {'loss': 0.388, 'grad_norm': 11.731395721435547, 'learning_rate': 9.066605692100155e-07, 'epoch': 0.24844046650393273, 'num_input_tokens_seen': 1920991232, 'completed': '24.84% (916 / 3_687)', 'remaining time': '24:46:43', 'throughput': '8095.84', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:33:50,884 >> {'loss': 0.3481, 'grad_norm': 11.196650505065918, 'learning_rate': 9.064142689966397e-07, 'epoch': 0.2487116897206401, 'num_input_tokens_seen': 1923088384, 'completed': '24.87% (917 / 3_687)', 'remaining time': '24:46:09', 'throughput': '8343.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:34:18,317 >> {'loss': 0.863, 'grad_norm': 21.21782684326172, 'learning_rate': 9.061676819546229e-07, 'epoch': 0.24898291293734745, 'num_input_tokens_seen': 1925185536, 'completed': '24.90% (918 / 3_687)', 'remaining time': '24:45:22', 'throughput': '9555.59', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:34:48,835 >> {'loss': 0.5098, 'grad_norm': 14.912806510925293, 'learning_rate': 9.059208082824087e-07, 'epoch': 0.24925413615405478, 'num_input_tokens_seen': 1927282688, 'completed': '24.93% (919 / 3_687)', 'remaining time': '24:44:45', 'throughput': '8589.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:35:19,428 >> {'loss': 0.6095, 'grad_norm': 18.304712295532227, 'learning_rate': 9.05673648178672e-07, 'epoch': 0.24952535937076215, 'num_input_tokens_seen': 1929379840, 'completed': '24.95% (920 / 3_687)', 'remaining time': '24:44:08', 'throughput': '8568.68', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:35:51,360 >> {'loss': 0.3207, 'grad_norm': 11.65080738067627, 'learning_rate': 9.054262018423175e-07, 'epoch': 0.24979658258746948, 'num_input_tokens_seen': 1931476992, 'completed': '24.98% (921 / 3_687)', 'remaining time': '24:43:35', 'throughput': '8209.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:36:20,441 >> {'loss': 0.8062, 'grad_norm': 21.233457565307617, 'learning_rate': 9.051784694724808e-07, 'epoch': 0.2500678058041768, 'num_input_tokens_seen': 1933574144, 'completed': '25.01% (922 / 3_687)', 'remaining time': '24:42:53', 'throughput': '9014.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:36:52,120 >> {'loss': 0.2804, 'grad_norm': 9.506631851196289, 'learning_rate': 9.049304512685274e-07, 'epoch': 0.2503390290208842, 'num_input_tokens_seen': 1935671296, 'completed': '25.03% (923 / 3_687)', 'remaining time': '24:42:20', 'throughput': '8274.94', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:37:21,589 >> {'loss': 0.3033, 'grad_norm': 10.630436897277832, 'learning_rate': 9.046821474300527e-07, 'epoch': 0.25061025223759154, 'num_input_tokens_seen': 1937768448, 'completed': '25.06% (924 / 3_687)', 'remaining time': '24:41:39', 'throughput': '8895.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:37:52,258 >> {'loss': 0.6053, 'grad_norm': 18.508195877075195, 'learning_rate': 9.044335581568827e-07, 'epoch': 0.2508814754542989, 'num_input_tokens_seen': 1939865600, 'completed': '25.09% (925 / 3_687)', 'remaining time': '24:41:03', 'throughput': '8547.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:38:21,890 >> {'loss': 0.3725, 'grad_norm': 11.025355339050293, 'learning_rate': 9.041846836490723e-07, 'epoch': 0.25115269867100626, 'num_input_tokens_seen': 1941962752, 'completed': '25.12% (926 / 3_687)', 'remaining time': '24:40:23', 'throughput': '8846.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:38:54,402 >> {'loss': 0.4114, 'grad_norm': 13.680339813232422, 'learning_rate': 9.03935524106906e-07, 'epoch': 0.25142392188771356, 'num_input_tokens_seen': 1944059904, 'completed': '25.14% (927 / 3_687)', 'remaining time': '24:39:52', 'throughput': '8063.05', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:39:25,069 >> {'loss': 0.2863, 'grad_norm': 10.082228660583496, 'learning_rate': 9.036860797308984e-07, 'epoch': 0.2516951451044209, 'num_input_tokens_seen': 1946157056, 'completed': '25.17% (928 / 3_687)', 'remaining time': '24:39:15', 'throughput': '8548.15', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:39:53,085 >> {'loss': 0.452, 'grad_norm': 15.916239738464355, 'learning_rate': 9.034363507217925e-07, 'epoch': 0.2519663683211283, 'num_input_tokens_seen': 1948254208, 'completed': '25.20% (929 / 3_687)', 'remaining time': '24:38:31', 'throughput': '9356.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:40:22,341 >> {'loss': 0.6497, 'grad_norm': 20.79644012451172, 'learning_rate': 9.031863372805606e-07, 'epoch': 0.25223759153783565, 'num_input_tokens_seen': 1950351360, 'completed': '25.22% (930 / 3_687)', 'remaining time': '24:37:50', 'throughput': '8960.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:40:54,217 >> {'loss': 0.3006, 'grad_norm': 15.72608757019043, 'learning_rate': 9.029360396084043e-07, 'epoch': 0.252508814754543, 'num_input_tokens_seen': 1952448512, 'completed': '25.25% (931 / 3_687)', 'remaining time': '24:37:17', 'throughput': '8223.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:41:24,763 >> {'loss': 0.5086, 'grad_norm': 13.770834922790527, 'learning_rate': 9.026854579067537e-07, 'epoch': 0.2527800379712503, 'num_input_tokens_seen': 1954545664, 'completed': '25.28% (932 / 3_687)', 'remaining time': '24:36:40', 'throughput': '8582.13', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:41:56,408 >> {'loss': 0.513, 'grad_norm': 18.707565307617188, 'learning_rate': 9.024345923772671e-07, 'epoch': 0.2530512611879577, 'num_input_tokens_seen': 1956642816, 'completed': '25.31% (933 / 3_687)', 'remaining time': '24:36:06', 'throughput': '8283.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:42:26,904 >> {'loss': 0.336, 'grad_norm': 11.008857727050781, 'learning_rate': 9.021834432218317e-07, 'epoch': 0.25332248440466504, 'num_input_tokens_seen': 1958739968, 'completed': '25.33% (934 / 3_687)', 'remaining time': '24:35:29', 'throughput': '8596.14', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:42:59,348 >> {'loss': 0.3631, 'grad_norm': 13.265803337097168, 'learning_rate': 9.019320106425629e-07, 'epoch': 0.2535937076213724, 'num_input_tokens_seen': 1960837120, 'completed': '25.36% (935 / 3_687)', 'remaining time': '24:34:58', 'throughput': '8079.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:43:29,540 >> {'loss': 0.2662, 'grad_norm': 11.919256210327148, 'learning_rate': 9.016802948418038e-07, 'epoch': 0.25386493083807976, 'num_input_tokens_seen': 1962934272, 'completed': '25.39% (936 / 3_687)', 'remaining time': '24:34:20', 'throughput': '8682.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:43:57,848 >> {'loss': 0.5253, 'grad_norm': 13.845582962036133, 'learning_rate': 9.014282960221257e-07, 'epoch': 0.25413615405478707, 'num_input_tokens_seen': 1965031424, 'completed': '25.41% (937 / 3_687)', 'remaining time': '24:33:37', 'throughput': '9260.86', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:44:28,860 >> {'loss': 0.444, 'grad_norm': 14.255606651306152, 'learning_rate': 9.01176014386328e-07, 'epoch': 0.25440737727149443, 'num_input_tokens_seen': 1967128576, 'completed': '25.44% (938 / 3_687)', 'remaining time': '24:33:01', 'throughput': '8452.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:44:57,724 >> {'loss': 0.6345, 'grad_norm': 19.263900756835938, 'learning_rate': 9.009234501374371e-07, 'epoch': 0.2546786004882018, 'num_input_tokens_seen': 1969225728, 'completed': '25.47% (939 / 3_687)', 'remaining time': '24:32:19', 'throughput': '9082.12', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:45:30,540 >> {'loss': 0.5709, 'grad_norm': 21.17646026611328, 'learning_rate': 9.006706034787071e-07, 'epoch': 0.25494982370490915, 'num_input_tokens_seen': 1971322880, 'completed': '25.49% (940 / 3_687)', 'remaining time': '24:31:49', 'throughput': '7988.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:45:59,350 >> {'loss': 0.6367, 'grad_norm': 16.962255477905273, 'learning_rate': 9.004174746136196e-07, 'epoch': 0.2552210469216165, 'num_input_tokens_seen': 1973420032, 'completed': '25.52% (941 / 3_687)', 'remaining time': '24:31:07', 'throughput': '9099.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:46:34,337 >> {'loss': 0.4449, 'grad_norm': 12.953526496887207, 'learning_rate': 9.001640637458829e-07, 'epoch': 0.2554922701383238, 'num_input_tokens_seen': 1975517184, 'completed': '25.55% (942 / 3_687)', 'remaining time': '24:30:43', 'throughput': '7492.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:47:03,644 >> {'loss': 0.9314, 'grad_norm': 19.991397857666016, 'learning_rate': 8.999103710794323e-07, 'epoch': 0.2557634933550312, 'num_input_tokens_seen': 1977614336, 'completed': '25.58% (943 / 3_687)', 'remaining time': '24:30:03', 'throughput': '8944.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:47:34,202 >> {'loss': 0.3151, 'grad_norm': 12.952409744262695, 'learning_rate': 8.996563968184302e-07, 'epoch': 0.25603471657173854, 'num_input_tokens_seen': 1979711488, 'completed': '25.60% (944 / 3_687)', 'remaining time': '24:29:26', 'throughput': '8578.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:48:05,265 >> {'loss': 0.7246, 'grad_norm': 18.413883209228516, 'learning_rate': 8.994021411672653e-07, 'epoch': 0.2563059397884459, 'num_input_tokens_seen': 1981808640, 'completed': '25.63% (945 / 3_687)', 'remaining time': '24:28:51', 'throughput': '8439.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:48:34,857 >> {'loss': 0.5387, 'grad_norm': 15.939887046813965, 'learning_rate': 8.99147604330553e-07, 'epoch': 0.25657716300515326, 'num_input_tokens_seen': 1983905792, 'completed': '25.66% (946 / 3_687)', 'remaining time': '24:28:11', 'throughput': '8858.61', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:49:07,911 >> {'loss': 0.5397, 'grad_norm': 18.361181259155273, 'learning_rate': 8.988927865131347e-07, 'epoch': 0.25684838622186057, 'num_input_tokens_seen': 1986002944, 'completed': '25.68% (947 / 3_687)', 'remaining time': '24:27:42', 'throughput': '7930.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:49:40,540 >> {'loss': 0.5404, 'grad_norm': 16.950855255126953, 'learning_rate': 8.986376879200783e-07, 'epoch': 0.25711960943856793, 'num_input_tokens_seen': 1988100096, 'completed': '25.71% (948 / 3_687)', 'remaining time': '24:27:11', 'throughput': '8034.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:50:12,002 >> {'loss': 0.3202, 'grad_norm': 13.930158615112305, 'learning_rate': 8.983823087566772e-07, 'epoch': 0.2573908326552753, 'num_input_tokens_seen': 1990197248, 'completed': '25.74% (949 / 3_687)', 'remaining time': '24:26:37', 'throughput': '8332.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:50:43,428 >> {'loss': 0.5814, 'grad_norm': 18.551612854003906, 'learning_rate': 8.981266492284511e-07, 'epoch': 0.25766205587198265, 'num_input_tokens_seen': 1992294400, 'completed': '25.77% (950 / 3_687)', 'remaining time': '24:26:03', 'throughput': '8341.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:51:15,813 >> {'loss': 0.6843, 'grad_norm': 17.812480926513672, 'learning_rate': 8.978707095411446e-07, 'epoch': 0.25793327908869, 'num_input_tokens_seen': 1994391552, 'completed': '25.79% (951 / 3_687)', 'remaining time': '24:25:31', 'throughput': '8094.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:51:44,274 >> {'loss': 1.0731, 'grad_norm': 22.378063201904297, 'learning_rate': 8.976144899007288e-07, 'epoch': 0.2582045023053973, 'num_input_tokens_seen': 1996488704, 'completed': '25.82% (952 / 3_687)', 'remaining time': '24:24:49', 'throughput': '9210.36', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:52:15,381 >> {'loss': 0.4466, 'grad_norm': 12.730658531188965, 'learning_rate': 8.973579905133991e-07, 'epoch': 0.2584757255221047, 'num_input_tokens_seen': 1998585856, 'completed': '25.85% (953 / 3_687)', 'remaining time': '24:24:14', 'throughput': '8427.31', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:52:45,893 >> {'loss': 0.2807, 'grad_norm': 9.290279388427734, 'learning_rate': 8.971012115855766e-07, 'epoch': 0.25874694873881204, 'num_input_tokens_seen': 2000683008, 'completed': '25.87% (954 / 3_687)', 'remaining time': '24:23:37', 'throughput': '8591.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:53:15,554 >> {'loss': 0.5023, 'grad_norm': 14.961581230163574, 'learning_rate': 8.968441533239073e-07, 'epoch': 0.2590181719555194, 'num_input_tokens_seen': 2002780160, 'completed': '25.90% (955 / 3_687)', 'remaining time': '24:22:58', 'throughput': '8837.96', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:53:46,683 >> {'loss': 0.3769, 'grad_norm': 11.520310401916504, 'learning_rate': 8.965868159352616e-07, 'epoch': 0.25928939517222677, 'num_input_tokens_seen': 2004877312, 'completed': '25.93% (956 / 3_687)', 'remaining time': '24:22:23', 'throughput': '8421.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:54:17,756 >> {'loss': 0.4165, 'grad_norm': 13.574871063232422, 'learning_rate': 8.963291996267354e-07, 'epoch': 0.2595606183889341, 'num_input_tokens_seen': 2006974464, 'completed': '25.96% (957 / 3_687)', 'remaining time': '24:21:48', 'throughput': '8436.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:54:50,535 >> {'loss': 0.3482, 'grad_norm': 10.172213554382324, 'learning_rate': 8.960713046056478e-07, 'epoch': 0.25983184160564143, 'num_input_tokens_seen': 2009071616, 'completed': '25.98% (958 / 3_687)', 'remaining time': '24:21:17', 'throughput': '7997.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:55:19,084 >> {'loss': 0.4703, 'grad_norm': 15.749048233032227, 'learning_rate': 8.958131310795434e-07, 'epoch': 0.2601030648223488, 'num_input_tokens_seen': 2011168768, 'completed': '26.01% (959 / 3_687)', 'remaining time': '24:20:35', 'throughput': '9182.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:55:49,842 >> {'loss': 0.3705, 'grad_norm': 13.04376220703125, 'learning_rate': 8.955546792561902e-07, 'epoch': 0.26037428803905616, 'num_input_tokens_seen': 2013265920, 'completed': '26.04% (960 / 3_687)', 'remaining time': '24:19:59', 'throughput': '8522.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:56:20,024 >> {'loss': 0.4021, 'grad_norm': 13.472237586975098, 'learning_rate': 8.952959493435806e-07, 'epoch': 0.2606455112557635, 'num_input_tokens_seen': 2015363072, 'completed': '26.06% (961 / 3_687)', 'remaining time': '24:19:21', 'throughput': '8685.30', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:56:49,409 >> {'loss': 0.5569, 'grad_norm': 15.37488079071045, 'learning_rate': 8.950369415499304e-07, 'epoch': 0.2609167344724708, 'num_input_tokens_seen': 2017460224, 'completed': '26.09% (962 / 3_687)', 'remaining time': '24:18:41', 'throughput': '8921.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:57:24,090 >> {'loss': 0.7467, 'grad_norm': 18.64729881286621, 'learning_rate': 8.947776560836793e-07, 'epoch': 0.2611879576891782, 'num_input_tokens_seen': 2019557376, 'completed': '26.12% (963 / 3_687)', 'remaining time': '24:18:17', 'throughput': '7558.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:57:55,648 >> {'loss': 0.4492, 'grad_norm': 13.231241226196289, 'learning_rate': 8.945180931534902e-07, 'epoch': 0.26145918090588555, 'num_input_tokens_seen': 2021654528, 'completed': '26.15% (964 / 3_687)', 'remaining time': '24:17:43', 'throughput': '8306.75', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:58:28,596 >> {'loss': 0.387, 'grad_norm': 13.122672080993652, 'learning_rate': 8.942582529682496e-07, 'epoch': 0.2617304041225929, 'num_input_tokens_seen': 2023751680, 'completed': '26.17% (965 / 3_687)', 'remaining time': '24:17:13', 'throughput': '7956.32', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:58:59,170 >> {'loss': 0.3468, 'grad_norm': 12.181504249572754, 'learning_rate': 8.939981357370672e-07, 'epoch': 0.26200162733930027, 'num_input_tokens_seen': 2025848832, 'completed': '26.20% (966 / 3_687)', 'remaining time': '24:16:37', 'throughput': '8574.08', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 19:59:30,649 >> {'loss': 0.4132, 'grad_norm': 13.75951862335205, 'learning_rate': 8.937377416692752e-07, 'epoch': 0.2622728505560076, 'num_input_tokens_seen': 2027945984, 'completed': '26.23% (967 / 3_687)', 'remaining time': '24:16:03', 'throughput': '8327.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:00:01,358 >> {'loss': 0.5208, 'grad_norm': 15.17529582977295, 'learning_rate': 8.934770709744289e-07, 'epoch': 0.26254407377271494, 'num_input_tokens_seen': 2030043136, 'completed': '26.25% (968 / 3_687)', 'remaining time': '24:15:27', 'throughput': '8536.52', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:00:34,277 >> {'loss': 0.3606, 'grad_norm': 12.855225563049316, 'learning_rate': 8.93216123862306e-07, 'epoch': 0.2628152969894223, 'num_input_tokens_seen': 2032140288, 'completed': '26.28% (969 / 3_687)', 'remaining time': '24:14:57', 'throughput': '7963.29', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:01:01,968 >> {'loss': 1.1691, 'grad_norm': 23.8533992767334, 'learning_rate': 8.929549005429071e-07, 'epoch': 0.26308652020612966, 'num_input_tokens_seen': 2034237440, 'completed': '26.31% (970 / 3_687)', 'remaining time': '24:14:12', 'throughput': '9466.71', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:01:32,651 >> {'loss': 0.4991, 'grad_norm': 12.444233894348145, 'learning_rate': 8.926934012264546e-07, 'epoch': 0.263357743422837, 'num_input_tokens_seen': 2036334592, 'completed': '26.34% (971 / 3_687)', 'remaining time': '24:13:36', 'throughput': '8543.64', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:02:04,828 >> {'loss': 0.603, 'grad_norm': 19.38515853881836, 'learning_rate': 8.924316261233933e-07, 'epoch': 0.2636289666395443, 'num_input_tokens_seen': 2038431744, 'completed': '26.36% (972 / 3_687)', 'remaining time': '24:13:04', 'throughput': '8146.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:02:36,393 >> {'loss': 0.4341, 'grad_norm': 25.431591033935547, 'learning_rate': 8.921695754443898e-07, 'epoch': 0.2639001898562517, 'num_input_tokens_seen': 2040528896, 'completed': '26.39% (973 / 3_687)', 'remaining time': '24:12:31', 'throughput': '8304.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:03:05,913 >> {'loss': 0.4873, 'grad_norm': 14.830904960632324, 'learning_rate': 8.919072494003325e-07, 'epoch': 0.26417141307295905, 'num_input_tokens_seen': 2042626048, 'completed': '26.42% (974 / 3_687)', 'remaining time': '24:11:51', 'throughput': '8880.28', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:03:39,599 >> {'loss': 0.386, 'grad_norm': 10.834352493286133, 'learning_rate': 8.916446482023313e-07, 'epoch': 0.2644426362896664, 'num_input_tokens_seen': 2044723200, 'completed': '26.44% (975 / 3_687)', 'remaining time': '24:11:23', 'throughput': '7781.97', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:04:09,485 >> {'loss': 0.5712, 'grad_norm': 18.31397819519043, 'learning_rate': 8.913817720617178e-07, 'epoch': 0.26471385950637377, 'num_input_tokens_seen': 2046820352, 'completed': '26.47% (976 / 3_687)', 'remaining time': '24:10:45', 'throughput': '8771.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:04:42,252 >> {'loss': 0.3854, 'grad_norm': 14.418971061706543, 'learning_rate': 8.911186211900448e-07, 'epoch': 0.2649850827230811, 'num_input_tokens_seen': 2048917504, 'completed': '26.50% (977 / 3_687)', 'remaining time': '24:10:15', 'throughput': '8000.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:05:15,680 >> {'loss': 0.6277, 'grad_norm': 15.985840797424316, 'learning_rate': 8.908551957990858e-07, 'epoch': 0.26525630593978844, 'num_input_tokens_seen': 2051014656, 'completed': '26.53% (978 / 3_687)', 'remaining time': '24:09:46', 'throughput': '7842.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:05:45,385 >> {'loss': 0.6733, 'grad_norm': 20.523096084594727, 'learning_rate': 8.90591496100836e-07, 'epoch': 0.2655275291564958, 'num_input_tokens_seen': 2053111808, 'completed': '26.55% (979 / 3_687)', 'remaining time': '24:09:08', 'throughput': '8825.04', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:06:15,294 >> {'loss': 0.5786, 'grad_norm': 21.447572708129883, 'learning_rate': 8.903275223075104e-07, 'epoch': 0.26579875237320316, 'num_input_tokens_seen': 2055208960, 'completed': '26.58% (980 / 3_687)', 'remaining time': '24:08:30', 'throughput': '8764.55', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:06:44,312 >> {'loss': 0.6692, 'grad_norm': 15.837870597839355, 'learning_rate': 8.900632746315454e-07, 'epoch': 0.2660699755899105, 'num_input_tokens_seen': 2057306112, 'completed': '26.61% (981 / 3_687)', 'remaining time': '24:07:49', 'throughput': '9034.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:07:13,438 >> {'loss': 1.0872, 'grad_norm': 25.582645416259766, 'learning_rate': 8.897987532855972e-07, 'epoch': 0.26634119880661783, 'num_input_tokens_seen': 2059403264, 'completed': '26.63% (982 / 3_687)', 'remaining time': '24:07:09', 'throughput': '9000.20', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:07:47,471 >> {'loss': 0.5819, 'grad_norm': 17.301437377929688, 'learning_rate': 8.895339584825429e-07, 'epoch': 0.2666124220233252, 'num_input_tokens_seen': 2061500416, 'completed': '26.66% (983 / 3_687)', 'remaining time': '24:06:42', 'throughput': '7702.74', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:08:21,393 >> {'loss': 0.4918, 'grad_norm': 14.604111671447754, 'learning_rate': 8.892688904354787e-07, 'epoch': 0.26688364524003255, 'num_input_tokens_seen': 2063597568, 'completed': '26.69% (984 / 3_687)', 'remaining time': '24:06:15', 'throughput': '7727.83', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:08:51,210 >> {'loss': 0.388, 'grad_norm': 12.52351188659668, 'learning_rate': 8.890035493577219e-07, 'epoch': 0.2671548684567399, 'num_input_tokens_seen': 2065694720, 'completed': '26.72% (985 / 3_687)', 'remaining time': '24:05:36', 'throughput': '8791.56', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:09:21,294 >> {'loss': 0.9703, 'grad_norm': 21.226011276245117, 'learning_rate': 8.887379354628085e-07, 'epoch': 0.2674260916734473, 'num_input_tokens_seen': 2067791872, 'completed': '26.74% (986 / 3_687)', 'remaining time': '24:04:59', 'throughput': '8713.76', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:09:50,655 >> {'loss': 0.36, 'grad_norm': 12.156089782714844, 'learning_rate': 8.884720489644945e-07, 'epoch': 0.2676973148901546, 'num_input_tokens_seen': 2069889024, 'completed': '26.77% (987 / 3_687)', 'remaining time': '24:04:19', 'throughput': '8928.50', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:10:18,445 >> {'loss': 0.6824, 'grad_norm': 16.55057144165039, 'learning_rate': 8.882058900767555e-07, 'epoch': 0.26796853810686194, 'num_input_tokens_seen': 2071986176, 'completed': '26.80% (988 / 3_687)', 'remaining time': '24:03:35', 'throughput': '9433.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:10:54,197 >> {'loss': 0.6588, 'grad_norm': 19.046653747558594, 'learning_rate': 8.879394590137857e-07, 'epoch': 0.2682397613235693, 'num_input_tokens_seen': 2074083328, 'completed': '26.82% (989 / 3_687)', 'remaining time': '24:03:13', 'throughput': '7332.22', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:11:27,337 >> {'loss': 0.3678, 'grad_norm': 13.83050537109375, 'learning_rate': 8.876727559899989e-07, 'epoch': 0.26851098454027666, 'num_input_tokens_seen': 2076180480, 'completed': '26.85% (990 / 3_687)', 'remaining time': '24:02:44', 'throughput': '7910.10', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:11:55,110 >> {'loss': 0.9572, 'grad_norm': 23.640356063842773, 'learning_rate': 8.874057812200274e-07, 'epoch': 0.26878220775698397, 'num_input_tokens_seen': 2078277632, 'completed': '26.88% (991 / 3_687)', 'remaining time': '24:02:00', 'throughput': '9438.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:12:26,384 >> {'loss': 0.722, 'grad_norm': 15.057960510253906, 'learning_rate': 8.871385349187225e-07, 'epoch': 0.26905343097369133, 'num_input_tokens_seen': 2080374784, 'completed': '26.91% (992 / 3_687)', 'remaining time': '24:01:26', 'throughput': '8382.42', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:12:56,729 >> {'loss': 0.7448, 'grad_norm': 18.069377899169922, 'learning_rate': 8.868710173011538e-07, 'epoch': 0.2693246541903987, 'num_input_tokens_seen': 2082471936, 'completed': '26.93% (993 / 3_687)', 'remaining time': '24:00:49', 'throughput': '8638.81', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:13:28,142 >> {'loss': 0.5023, 'grad_norm': 13.327810287475586, 'learning_rate': 8.866032285826091e-07, 'epoch': 0.26959587740710605, 'num_input_tokens_seen': 2084569088, 'completed': '26.96% (994 / 3_687)', 'remaining time': '24:00:15', 'throughput': '8344.90', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:14:01,370 >> {'loss': 0.357, 'grad_norm': 11.229975700378418, 'learning_rate': 8.86335168978595e-07, 'epoch': 0.2698671006238134, 'num_input_tokens_seen': 2086666240, 'completed': '26.99% (995 / 3_687)', 'remaining time': '23:59:46', 'throughput': '7889.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:14:33,944 >> {'loss': 0.2903, 'grad_norm': 10.154101371765137, 'learning_rate': 8.860668387048353e-07, 'epoch': 0.2701383238405207, 'num_input_tokens_seen': 2088763392, 'completed': '27.01% (996 / 3_687)', 'remaining time': '23:59:15', 'throughput': '8047.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:15:04,744 >> {'loss': 0.3106, 'grad_norm': 12.260966300964355, 'learning_rate': 8.85798237977272e-07, 'epoch': 0.2704095470572281, 'num_input_tokens_seen': 2090860544, 'completed': '27.04% (997 / 3_687)', 'remaining time': '23:58:40', 'throughput': '8511.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:15:38,082 >> {'loss': 0.6471, 'grad_norm': 18.005229949951172, 'learning_rate': 8.85529367012065e-07, 'epoch': 0.27068077027393544, 'num_input_tokens_seen': 2092957696, 'completed': '27.07% (998 / 3_687)', 'remaining time': '23:58:11', 'throughput': '7863.11', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:16:09,683 >> {'loss': 0.4971, 'grad_norm': 13.38134765625, 'learning_rate': 8.852602260255911e-07, 'epoch': 0.2709519934906428, 'num_input_tokens_seen': 2095054848, 'completed': '27.10% (999 / 3_687)', 'remaining time': '23:57:37', 'throughput': '8295.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:16:41,488 >> {'loss': 0.7469, 'grad_norm': 17.00292205810547, 'learning_rate': 8.849908152344451e-07, 'epoch': 0.27122321670735017, 'num_input_tokens_seen': 2097152000, 'completed': '27.12% (1_000 / 3_687)', 'remaining time': '23:57:05', 'throughput': '8242.24', 'gpu_mem_free': '5581MB'} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[INFO|trainer.py:3503] 2025-01-01 20:17:05,871 >> Saving model checkpoint to /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000 +[INFO|configuration_utils.py:472] 2025-01-01 20:17:05,877 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/config.json +[INFO|configuration_utils.py:807] 2025-01-01 20:17:05,878 >> Configuration saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/generation_config.json +[INFO|modeling_utils.py:2807] 2025-01-01 20:18:02,543 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 7 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/model.safetensors.index.json. +[INFO|tokenization_utils_base.py:2684] 2025-01-01 20:18:02,546 >> tokenizer config file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/tokenizer_config.json +[INFO|tokenization_utils_base.py:2693] 2025-01-01 20:18:02,547 >> Special tokens file saved in /scratch3/workspace/ctpham_umass_edu-ft/_llama-3.1-8b-instruct_bsz-16_lr-1e-6_epochs-1_/checkpoint-1000/special_tokens_map.json +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:689: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( +[WARNING|trainer.py:868] 2025-01-01 20:21:48,379 >> Save streaming dataset state: {'epoch': 0, 'sample_in_epoch': 2000, 'num_canonical_nodes': 1, 'shuffle_seed': 42, 'initial_physical_nodes': 1} +/scratch3/workspace/ctpham_umass_edu-ft/envs/prolong-final/lib/python3.10/site-packages/torch/utils/checkpoint.py:1399: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead. + with device_autocast_ctx, torch.cpu.amp.autocast(**cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] +[INFO|trainer.py:175] 2025-01-01 20:22:18,860 >> {'loss': 0.3657, 'grad_norm': 14.504280090332031, 'learning_rate': 8.847211348554382e-07, 'epoch': 0.2714944399240575, 'num_input_tokens_seen': 2099249152, 'completed': '27.15% (1_001 / 3_687)', 'remaining time': '24:10:12', 'throughput': '777.02', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:22:51,020 >> {'loss': 0.428, 'grad_norm': 12.289542198181152, 'learning_rate': 8.844511851055991e-07, 'epoch': 0.27176566314076483, 'num_input_tokens_seen': 2101346304, 'completed': '27.18% (1_002 / 3_687)', 'remaining time': '24:09:39', 'throughput': '8151.23', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:23:22,307 >> {'loss': 0.4553, 'grad_norm': 13.12674617767334, 'learning_rate': 8.841809662021731e-07, 'epoch': 0.2720368863574722, 'num_input_tokens_seen': 2103443456, 'completed': '27.20% (1_003 / 3_687)', 'remaining time': '24:09:03', 'throughput': '8378.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:23:53,846 >> {'loss': 0.8248, 'grad_norm': 21.255281448364258, 'learning_rate': 8.839104783626219e-07, 'epoch': 0.27230810957417956, 'num_input_tokens_seen': 2105540608, 'completed': '27.23% (1_004 / 3_687)', 'remaining time': '24:08:29', 'throughput': '8311.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:24:24,224 >> {'loss': 0.363, 'grad_norm': 11.34033489227295, 'learning_rate': 8.836397218046239e-07, 'epoch': 0.2725793327908869, 'num_input_tokens_seen': 2107637760, 'completed': '27.26% (1_005 / 3_687)', 'remaining time': '24:07:51', 'throughput': '8629.65', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:24:53,842 >> {'loss': 0.4685, 'grad_norm': 15.393781661987305, 'learning_rate': 8.83368696746074e-07, 'epoch': 0.2728505560075942, 'num_input_tokens_seen': 2109734912, 'completed': '27.29% (1_006 / 3_687)', 'remaining time': '24:07:11', 'throughput': '8850.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:25:23,401 >> {'loss': 0.7723, 'grad_norm': 17.88239288330078, 'learning_rate': 8.830974034050824e-07, 'epoch': 0.2731217792243016, 'num_input_tokens_seen': 2111832064, 'completed': '27.31% (1_007 / 3_687)', 'remaining time': '24:06:31', 'throughput': '8868.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:25:52,708 >> {'loss': 0.2608, 'grad_norm': 8.97508430480957, 'learning_rate': 8.828258419999759e-07, 'epoch': 0.27339300244100895, 'num_input_tokens_seen': 2113929216, 'completed': '27.34% (1_008 / 3_687)', 'remaining time': '24:05:51', 'throughput': '8944.78', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:26:25,053 >> {'loss': 0.2842, 'grad_norm': 9.696796417236328, 'learning_rate': 8.825540127492965e-07, 'epoch': 0.2736642256577163, 'num_input_tokens_seen': 2116026368, 'completed': '27.37% (1_009 / 3_687)', 'remaining time': '24:05:18', 'throughput': '8104.48', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:26:56,955 >> {'loss': 0.7218, 'grad_norm': 19.162878036499023, 'learning_rate': 8.822819158718026e-07, 'epoch': 0.27393544887442367, 'num_input_tokens_seen': 2118123520, 'completed': '27.39% (1_010 / 3_687)', 'remaining time': '24:04:44', 'throughput': '8217.49', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:27:28,594 >> {'loss': 0.4699, 'grad_norm': 16.030359268188477, 'learning_rate': 8.820095515864669e-07, 'epoch': 0.274206672091131, 'num_input_tokens_seen': 2120220672, 'completed': '27.42% (1_011 / 3_687)', 'remaining time': '24:04:10', 'throughput': '8285.09', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:27:59,270 >> {'loss': 0.5692, 'grad_norm': 18.105058670043945, 'learning_rate': 8.81736920112478e-07, 'epoch': 0.27447789530783834, 'num_input_tokens_seen': 2122317824, 'completed': '27.45% (1_012 / 3_687)', 'remaining time': '24:03:33', 'throughput': '8545.82', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:28:34,604 >> {'loss': 0.45, 'grad_norm': 15.880881309509277, 'learning_rate': 8.814640216692391e-07, 'epoch': 0.2747491185245457, 'num_input_tokens_seen': 2124414976, 'completed': '27.47% (1_013 / 3_687)', 'remaining time': '24:03:09', 'throughput': '7419.03', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:29:02,820 >> {'loss': 0.8558, 'grad_norm': 26.699430465698242, 'learning_rate': 8.81190856476369e-07, 'epoch': 0.27502034174125306, 'num_input_tokens_seen': 2126512128, 'completed': '27.50% (1_014 / 3_687)', 'remaining time': '24:02:25', 'throughput': '9290.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:29:32,656 >> {'loss': 0.6491, 'grad_norm': 20.639781951904297, 'learning_rate': 8.809174247537003e-07, 'epoch': 0.2752915649579604, 'num_input_tokens_seen': 2128609280, 'completed': '27.53% (1_015 / 3_687)', 'remaining time': '24:01:46', 'throughput': '8786.21', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:30:05,393 >> {'loss': 0.2257, 'grad_norm': 9.554243087768555, 'learning_rate': 8.806437267212805e-07, 'epoch': 0.2755627881746677, 'num_input_tokens_seen': 2130706432, 'completed': '27.56% (1_016 / 3_687)', 'remaining time': '24:01:15', 'throughput': '8007.70', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:30:39,554 >> {'loss': 0.8779, 'grad_norm': 38.22649383544922, 'learning_rate': 8.803697625993713e-07, 'epoch': 0.2758340113913751, 'num_input_tokens_seen': 2132803584, 'completed': '27.58% (1_017 / 3_687)', 'remaining time': '24:00:47', 'throughput': '7673.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:31:08,917 >> {'loss': 0.6113, 'grad_norm': 15.548166275024414, 'learning_rate': 8.800955326084487e-07, 'epoch': 0.27610523460808245, 'num_input_tokens_seen': 2134900736, 'completed': '27.61% (1_018 / 3_687)', 'remaining time': '24:00:07', 'throughput': '8927.87', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:31:38,562 >> {'loss': 0.2526, 'grad_norm': 9.940999031066895, 'learning_rate': 8.798210369692025e-07, 'epoch': 0.2763764578247898, 'num_input_tokens_seen': 2136997888, 'completed': '27.64% (1_019 / 3_687)', 'remaining time': '23:59:27', 'throughput': '8842.69', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:32:08,637 >> {'loss': 0.5191, 'grad_norm': 13.762343406677246, 'learning_rate': 8.795462759025364e-07, 'epoch': 0.2766476810414972, 'num_input_tokens_seen': 2139095040, 'completed': '27.66% (1_020 / 3_687)', 'remaining time': '23:58:49', 'throughput': '8716.41', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:32:41,131 >> {'loss': 0.3506, 'grad_norm': 12.772849082946777, 'learning_rate': 8.792712496295677e-07, 'epoch': 0.2769189042582045, 'num_input_tokens_seen': 2141192192, 'completed': '27.69% (1_021 / 3_687)', 'remaining time': '23:58:17', 'throughput': '8067.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:33:12,105 >> {'loss': 0.4284, 'grad_norm': 13.662134170532227, 'learning_rate': 8.789959583716268e-07, 'epoch': 0.27719012747491184, 'num_input_tokens_seen': 2143289344, 'completed': '27.72% (1_022 / 3_687)', 'remaining time': '23:57:41', 'throughput': '8463.37', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:33:42,439 >> {'loss': 0.3021, 'grad_norm': 11.673541069030762, 'learning_rate': 8.787204023502579e-07, 'epoch': 0.2774613506916192, 'num_input_tokens_seen': 2145386496, 'completed': '27.75% (1_023 / 3_687)', 'remaining time': '23:57:03', 'throughput': '8641.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:34:10,026 >> {'loss': 0.4575, 'grad_norm': 13.588297843933105, 'learning_rate': 8.78444581787218e-07, 'epoch': 0.27773257390832656, 'num_input_tokens_seen': 2147483648, 'completed': '27.77% (1_024 / 3_687)', 'remaining time': '23:56:18', 'throughput': '9502.51', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:34:42,816 >> {'loss': 0.5034, 'grad_norm': 13.916876792907715, 'learning_rate': 8.781684969044769e-07, 'epoch': 0.2780037971250339, 'num_input_tokens_seen': 2149580800, 'completed': '27.80% (1_025 / 3_687)', 'remaining time': '23:55:47', 'throughput': '7994.77', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:35:15,225 >> {'loss': 0.5808, 'grad_norm': 17.712745666503906, 'learning_rate': 8.778921479242173e-07, 'epoch': 0.27827502034174123, 'num_input_tokens_seen': 2151677952, 'completed': '27.83% (1_026 / 3_687)', 'remaining time': '23:55:15', 'throughput': '8088.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:35:46,119 >> {'loss': 0.2335, 'grad_norm': 11.685976028442383, 'learning_rate': 8.776155350688342e-07, 'epoch': 0.2785462435584486, 'num_input_tokens_seen': 2153775104, 'completed': '27.85% (1_027 / 3_687)', 'remaining time': '23:54:39', 'throughput': '8485.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:36:16,705 >> {'loss': 0.2984, 'grad_norm': 12.935453414916992, 'learning_rate': 8.773386585609352e-07, 'epoch': 0.27881746677515595, 'num_input_tokens_seen': 2155872256, 'completed': '27.88% (1_028 / 3_687)', 'remaining time': '23:54:02', 'throughput': '8570.80', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:36:46,676 >> {'loss': 0.4567, 'grad_norm': 13.87713623046875, 'learning_rate': 8.770615186233398e-07, 'epoch': 0.2790886899918633, 'num_input_tokens_seen': 2157969408, 'completed': '27.91% (1_029 / 3_687)', 'remaining time': '23:53:23', 'throughput': '8746.54', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:37:17,744 >> {'loss': 0.3844, 'grad_norm': 13.851126670837402, 'learning_rate': 8.7678411547908e-07, 'epoch': 0.2793599132085707, 'num_input_tokens_seen': 2160066560, 'completed': '27.94% (1_030 / 3_687)', 'remaining time': '23:52:48', 'throughput': '8437.91', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:37:47,124 >> {'loss': 0.8872, 'grad_norm': 19.612043380737305, 'learning_rate': 8.76506449351399e-07, 'epoch': 0.279631136425278, 'num_input_tokens_seen': 2162163712, 'completed': '27.96% (1_031 / 3_687)', 'remaining time': '23:52:08', 'throughput': '8922.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:38:20,052 >> {'loss': 0.3742, 'grad_norm': 23.587507247924805, 'learning_rate': 8.762285204637522e-07, 'epoch': 0.27990235964198534, 'num_input_tokens_seen': 2164260864, 'completed': '27.99% (1_032 / 3_687)', 'remaining time': '23:51:37', 'throughput': '7961.24', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:38:48,608 >> {'loss': 0.9539, 'grad_norm': 21.63985252380371, 'learning_rate': 8.75950329039806e-07, 'epoch': 0.2801735828586927, 'num_input_tokens_seen': 2166358016, 'completed': '28.02% (1_033 / 3_687)', 'remaining time': '23:50:55', 'throughput': '9180.07', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:39:20,389 >> {'loss': 0.4595, 'grad_norm': 14.81737232208252, 'learning_rate': 8.756718753034381e-07, 'epoch': 0.28044480607540007, 'num_input_tokens_seen': 2168455168, 'completed': '28.04% (1_034 / 3_687)', 'remaining time': '23:50:21', 'throughput': '8248.26', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:39:52,304 >> {'loss': 0.41, 'grad_norm': 11.914786338806152, 'learning_rate': 8.75393159478738e-07, 'epoch': 0.2807160292921074, 'num_input_tokens_seen': 2170552320, 'completed': '28.07% (1_035 / 3_687)', 'remaining time': '23:49:47', 'throughput': '8214.01', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:40:23,342 >> {'loss': 0.5702, 'grad_norm': 15.636473655700684, 'learning_rate': 8.751141817900052e-07, 'epoch': 0.28098725250881473, 'num_input_tokens_seen': 2172649472, 'completed': '28.10% (1_036 / 3_687)', 'remaining time': '23:49:12', 'throughput': '8445.67', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:40:56,975 >> {'loss': 0.3074, 'grad_norm': 11.957223892211914, 'learning_rate': 8.748349424617504e-07, 'epoch': 0.2812584757255221, 'num_input_tokens_seen': 2174746624, 'completed': '28.13% (1_037 / 3_687)', 'remaining time': '23:48:43', 'throughput': '7794.43', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:41:29,585 >> {'loss': 0.7127, 'grad_norm': 18.70858383178711, 'learning_rate': 8.745554417186946e-07, 'epoch': 0.28152969894222946, 'num_input_tokens_seen': 2176843776, 'completed': '28.15% (1_038 / 3_687)', 'remaining time': '23:48:11', 'throughput': '8038.72', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:41:59,434 >> {'loss': 0.7422, 'grad_norm': 19.931575775146484, 'learning_rate': 8.742756797857698e-07, 'epoch': 0.2818009221589368, 'num_input_tokens_seen': 2178940928, 'completed': '28.18% (1_039 / 3_687)', 'remaining time': '23:47:32', 'throughput': '8782.17', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:42:32,085 >> {'loss': 0.5126, 'grad_norm': 15.31186294555664, 'learning_rate': 8.739956568881174e-07, 'epoch': 0.2820721453756442, 'num_input_tokens_seen': 2181038080, 'completed': '28.21% (1_040 / 3_687)', 'remaining time': '23:47:01', 'throughput': '8028.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:43:04,515 >> {'loss': 0.5143, 'grad_norm': 17.57640266418457, 'learning_rate': 8.737153732510894e-07, 'epoch': 0.2823433685923515, 'num_input_tokens_seen': 2183135232, 'completed': '28.23% (1_041 / 3_687)', 'remaining time': '23:46:28', 'throughput': '8083.19', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:43:34,257 >> {'loss': 0.3962, 'grad_norm': 11.209922790527344, 'learning_rate': 8.734348291002472e-07, 'epoch': 0.28261459180905885, 'num_input_tokens_seen': 2185232384, 'completed': '28.26% (1_042 / 3_687)', 'remaining time': '23:45:50', 'throughput': '8813.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:44:06,262 >> {'loss': 0.6749, 'grad_norm': 15.845287322998047, 'learning_rate': 8.731540246613621e-07, 'epoch': 0.2828858150257662, 'num_input_tokens_seen': 2187329536, 'completed': '28.29% (1_043 / 3_687)', 'remaining time': '23:45:16', 'throughput': '8190.92', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:44:37,809 >> {'loss': 0.2823, 'grad_norm': 10.98125171661377, 'learning_rate': 8.728729601604149e-07, 'epoch': 0.28315703824247357, 'num_input_tokens_seen': 2189426688, 'completed': '28.32% (1_044 / 3_687)', 'remaining time': '23:44:42', 'throughput': '8309.46', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:45:08,418 >> {'loss': 0.5347, 'grad_norm': 16.057796478271484, 'learning_rate': 8.725916358235956e-07, 'epoch': 0.28342826145918093, 'num_input_tokens_seen': 2191523840, 'completed': '28.34% (1_045 / 3_687)', 'remaining time': '23:44:05', 'throughput': '8564.39', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:45:35,577 >> {'loss': 0.7123, 'grad_norm': 15.23913860321045, 'learning_rate': 8.723100518773034e-07, 'epoch': 0.28369948467588824, 'num_input_tokens_seen': 2193620992, 'completed': '28.37% (1_046 / 3_687)', 'remaining time': '23:43:20', 'throughput': '9651.99', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:46:06,424 >> {'loss': 0.3393, 'grad_norm': 13.005721092224121, 'learning_rate': 8.720282085481463e-07, 'epoch': 0.2839707078925956, 'num_input_tokens_seen': 2195718144, 'completed': '28.40% (1_047 / 3_687)', 'remaining time': '23:42:44', 'throughput': '8498.25', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:46:38,374 >> {'loss': 0.4552, 'grad_norm': 12.790346145629883, 'learning_rate': 8.717461060629408e-07, 'epoch': 0.28424193110930296, 'num_input_tokens_seen': 2197815296, 'completed': '28.42% (1_048 / 3_687)', 'remaining time': '23:42:10', 'throughput': '8204.85', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:47:09,237 >> {'loss': 0.5892, 'grad_norm': 21.711978912353516, 'learning_rate': 8.714637446487127e-07, 'epoch': 0.2845131543260103, 'num_input_tokens_seen': 2199912448, 'completed': '28.45% (1_049 / 3_687)', 'remaining time': '23:41:34', 'throughput': '8493.73', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:47:39,231 >> {'loss': 0.468, 'grad_norm': 14.864642143249512, 'learning_rate': 8.711811245326955e-07, 'epoch': 0.2847843775427177, 'num_input_tokens_seen': 2202009600, 'completed': '28.48% (1_050 / 3_687)', 'remaining time': '23:40:56', 'throughput': '8740.06', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:48:12,428 >> {'loss': 0.5368, 'grad_norm': 15.85414981842041, 'learning_rate': 8.70898245942331e-07, 'epoch': 0.285055600759425, 'num_input_tokens_seen': 2204106752, 'completed': '28.51% (1_051 / 3_687)', 'remaining time': '23:40:26', 'throughput': '7896.63', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:48:42,229 >> {'loss': 0.5994, 'grad_norm': 17.953489303588867, 'learning_rate': 8.706151091052693e-07, 'epoch': 0.28532682397613235, 'num_input_tokens_seen': 2206203904, 'completed': '28.53% (1_052 / 3_687)', 'remaining time': '23:39:47', 'throughput': '8796.35', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:49:15,318 >> {'loss': 0.5299, 'grad_norm': 14.528735160827637, 'learning_rate': 8.703317142493681e-07, 'epoch': 0.2855980471928397, 'num_input_tokens_seen': 2208301056, 'completed': '28.56% (1_053 / 3_687)', 'remaining time': '23:39:17', 'throughput': '7922.62', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:49:44,022 >> {'loss': 0.6811, 'grad_norm': 17.70334815979004, 'learning_rate': 8.700480616026928e-07, 'epoch': 0.28586927040954707, 'num_input_tokens_seen': 2210398208, 'completed': '28.59% (1_054 / 3_687)', 'remaining time': '23:38:36', 'throughput': '9132.40', 'gpu_mem_free': '5581MB'} +[INFO|trainer.py:175] 2025-01-01 20:50:15,665 >> {'loss': 0.5513, 'grad_norm': 15.61217975616455, 'learning_rate': 8.697641513935164e-07, 'epoch': 0.28614049362625443, 'num_input_tokens_seen': 2212495360, 'completed': '28.61% (1_055 / 3_687)', 'remaining time': '23:38:01', 'throughput': '8284.46', 'gpu_mem_free': '5581MB'}