Training in progress, step 35758

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbab78eb88012b99b9f7beb85b1c99d10defaa1626416835c0fab109635ac9f9
+oid sha256:8d8d00a1f1e6be48f490027f06d8f0499a8da972f53e674aa40b182a688ad2a8
 size 3554214752

trainer_log.jsonl

@@ -3501,3 +3501,76 @@
 {"current_steps": 35010, "total_steps": 35758, "loss": 0.0053, "lr": 1.3323671442529994e-08, "epoch": 1.958108448222825, "percentage": 97.91, "elapsed_time": "1 day, 11:34:47", "remaining_time": "0:45:36"}
 {"current_steps": 35020, "total_steps": 35758, "loss": 0.0067, "lr": 1.2969957947551182e-08, "epoch": 1.9586677479795296, "percentage": 97.94, "elapsed_time": "1 day, 11:35:25", "remaining_time": "0:45:00"}
 {"current_steps": 35030, "total_steps": 35758, "loss": 0.0072, "lr": 1.262099688153995e-08, "epoch": 1.9592270477362344, "percentage": 97.96, "elapsed_time": "1 day, 11:35:57", "remaining_time": "0:44:23"}
+{"current_steps": 35040, "total_steps": 35758, "loss": 0.0058, "lr": 1.2276788577040289e-08, "epoch": 1.959786347492939, "percentage": 97.99, "elapsed_time": "1 day, 11:36:33", "remaining_time": "0:43:46"}
+{"current_steps": 35050, "total_steps": 35758, "loss": 0.0064, "lr": 1.1937333362068703e-08, "epoch": 1.9603456472496434, "percentage": 98.02, "elapsed_time": "1 day, 11:37:11", "remaining_time": "0:43:10"}
+{"current_steps": 35060, "total_steps": 35758, "loss": 0.0066, "lr": 1.1602631560111987e-08, "epoch": 1.9609049470063482, "percentage": 98.05, "elapsed_time": "1 day, 11:37:49", "remaining_time": "0:42:33"}
+{"current_steps": 35070, "total_steps": 35758, "loss": 0.0058, "lr": 1.1272683490126667e-08, "epoch": 1.9614642467630525, "percentage": 98.08, "elapsed_time": "1 day, 11:38:23", "remaining_time": "0:41:57"}
+{"current_steps": 35080, "total_steps": 35758, "loss": 0.0086, "lr": 1.0947489466539008e-08, "epoch": 1.9620235465197573, "percentage": 98.1, "elapsed_time": "1 day, 11:39:06", "remaining_time": "0:41:20"}
+{"current_steps": 35090, "total_steps": 35758, "loss": 0.0061, "lr": 1.0627049799245559e-08, "epoch": 1.9625828462764618, "percentage": 98.13, "elapsed_time": "1 day, 11:39:45", "remaining_time": "0:40:44"}
+{"current_steps": 35100, "total_steps": 35758, "loss": 0.0052, "lr": 1.03113647936115e-08, "epoch": 1.9631421460331664, "percentage": 98.16, "elapsed_time": "1 day, 11:40:18", "remaining_time": "0:40:07"}
+{"current_steps": 35110, "total_steps": 35758, "loss": 0.0064, "lr": 1.000043475047119e-08, "epoch": 1.9637014457898712, "percentage": 98.19, "elapsed_time": "1 day, 11:40:56", "remaining_time": "0:39:30"}
+{"current_steps": 35120, "total_steps": 35758, "loss": 0.0056, "lr": 9.69425996612816e-09, "epoch": 1.9642607455465757, "percentage": 98.22, "elapsed_time": "1 day, 11:41:30", "remaining_time": "0:38:54"}
+{"current_steps": 35130, "total_steps": 35758, "loss": 0.0057, "lr": 9.392840732354025e-09, "epoch": 1.9648200453032802, "percentage": 98.24, "elapsed_time": "1 day, 11:42:04", "remaining_time": "0:38:17"}
+{"current_steps": 35140, "total_steps": 35758, "loss": 0.0068, "lr": 9.096177336388457e-09, "epoch": 1.965379345059985, "percentage": 98.27, "elapsed_time": "1 day, 11:42:42", "remaining_time": "0:37:41"}
+{"current_steps": 35150, "total_steps": 35758, "loss": 0.0072, "lr": 8.804270060939202e-09, "epoch": 1.9659386448166893, "percentage": 98.3, "elapsed_time": "1 day, 11:43:19", "remaining_time": "0:37:04"}
+{"current_steps": 35160, "total_steps": 35758, "loss": 0.0067, "lr": 8.517119184181522e-09, "epoch": 1.966497944573394, "percentage": 98.33, "elapsed_time": "1 day, 11:43:54", "remaining_time": "0:36:27"}
+{"current_steps": 35170, "total_steps": 35758, "loss": 0.0072, "lr": 8.234724979758191e-09, "epoch": 1.9670572443300989, "percentage": 98.36, "elapsed_time": "1 day, 11:44:30", "remaining_time": "0:35:51"}
+{"current_steps": 35180, "total_steps": 35758, "loss": 0.0051, "lr": 7.957087716778945e-09, "epoch": 1.9676165440868032, "percentage": 98.38, "elapsed_time": "1 day, 11:45:02", "remaining_time": "0:35:14"}
+{"current_steps": 35190, "total_steps": 35758, "loss": 0.0061, "lr": 7.68420765981992e-09, "epoch": 1.968175843843508, "percentage": 98.41, "elapsed_time": "1 day, 11:45:37", "remaining_time": "0:34:37"}
+{"current_steps": 35200, "total_steps": 35758, "loss": 0.0052, "lr": 7.416085068924772e-09, "epoch": 1.9687351436002125, "percentage": 98.44, "elapsed_time": "1 day, 11:46:14", "remaining_time": "0:34:01"}
+{"current_steps": 35210, "total_steps": 35758, "loss": 0.0063, "lr": 7.152720199603003e-09, "epoch": 1.969294443356917, "percentage": 98.47, "elapsed_time": "1 day, 11:46:51", "remaining_time": "0:33:24"}
+{"current_steps": 35220, "total_steps": 35758, "loss": 0.0059, "lr": 6.894113302829409e-09, "epoch": 1.9698537431136218, "percentage": 98.5, "elapsed_time": "1 day, 11:47:29", "remaining_time": "0:32:48"}
+{"current_steps": 35230, "total_steps": 35758, "loss": 0.0063, "lr": 6.640264625046855e-09, "epoch": 1.9704130428703264, "percentage": 98.52, "elapsed_time": "1 day, 11:48:04", "remaining_time": "0:32:11"}
+{"current_steps": 35240, "total_steps": 35758, "loss": 0.0072, "lr": 6.3911744081612825e-09, "epoch": 1.970972342627031, "percentage": 98.55, "elapsed_time": "1 day, 11:48:36", "remaining_time": "0:31:34"}
+{"current_steps": 35250, "total_steps": 35758, "loss": 0.0057, "lr": 6.14684288954559e-09, "epoch": 1.9715316423837357, "percentage": 98.58, "elapsed_time": "1 day, 11:49:09", "remaining_time": "0:30:58"}
+{"current_steps": 35260, "total_steps": 35758, "loss": 0.0061, "lr": 5.907270302037416e-09, "epoch": 1.97209094214044, "percentage": 98.61, "elapsed_time": "1 day, 11:49:44", "remaining_time": "0:30:21"}
+{"current_steps": 35270, "total_steps": 35758, "loss": 0.0079, "lr": 5.672456873939136e-09, "epoch": 1.9726502418971448, "percentage": 98.64, "elapsed_time": "1 day, 11:50:25", "remaining_time": "0:29:45"}
+{"current_steps": 35280, "total_steps": 35758, "loss": 0.0068, "lr": 5.4424028290184225e-09, "epoch": 1.9732095416538495, "percentage": 98.66, "elapsed_time": "1 day, 11:50:59", "remaining_time": "0:29:08"}
+{"current_steps": 35290, "total_steps": 35758, "loss": 0.0071, "lr": 5.217108386506575e-09, "epoch": 1.9737688414105539, "percentage": 98.69, "elapsed_time": "1 day, 11:51:39", "remaining_time": "0:28:32"}
+{"current_steps": 35300, "total_steps": 35758, "loss": 0.0072, "lr": 4.996573761099632e-09, "epoch": 1.9743281411672586, "percentage": 98.72, "elapsed_time": "1 day, 11:52:17", "remaining_time": "0:27:55"}
+{"current_steps": 35310, "total_steps": 35758, "loss": 0.0051, "lr": 4.780799162958372e-09, "epoch": 1.9748874409239632, "percentage": 98.75, "elapsed_time": "1 day, 11:52:50", "remaining_time": "0:27:18"}
+{"current_steps": 35320, "total_steps": 35758, "loss": 0.0052, "lr": 4.56978479770609e-09, "epoch": 1.9754467406806677, "percentage": 98.78, "elapsed_time": "1 day, 11:53:26", "remaining_time": "0:26:42"}
+{"current_steps": 35330, "total_steps": 35758, "loss": 0.0061, "lr": 4.363530866431376e-09, "epoch": 1.9760060404373725, "percentage": 98.8, "elapsed_time": "1 day, 11:53:59", "remaining_time": "0:26:05"}
+{"current_steps": 35340, "total_steps": 35758, "loss": 0.0061, "lr": 4.162037565684784e-09, "epoch": 1.976565340194077, "percentage": 98.83, "elapsed_time": "1 day, 11:54:36", "remaining_time": "0:25:29"}
+{"current_steps": 35350, "total_steps": 35758, "loss": 0.0093, "lr": 3.965305087481608e-09, "epoch": 1.9771246399507816, "percentage": 98.86, "elapsed_time": "1 day, 11:55:14", "remaining_time": "0:24:52"}
+{"current_steps": 35360, "total_steps": 35758, "loss": 0.0049, "lr": 3.773333619298547e-09, "epoch": 1.9776839397074863, "percentage": 98.89, "elapsed_time": "1 day, 11:55:45", "remaining_time": "0:24:15"}
+{"current_steps": 35370, "total_steps": 35758, "loss": 0.0067, "lr": 3.5861233440775968e-09, "epoch": 1.9782432394641907, "percentage": 98.91, "elapsed_time": "1 day, 11:56:24", "remaining_time": "0:23:39"}
+{"current_steps": 35380, "total_steps": 35758, "loss": 0.0064, "lr": 3.403674440221605e-09, "epoch": 1.9788025392208954, "percentage": 98.94, "elapsed_time": "1 day, 11:56:58", "remaining_time": "0:23:02"}
+{"current_steps": 35390, "total_steps": 35758, "loss": 0.0065, "lr": 3.2259870815964935e-09, "epoch": 1.9793618389776002, "percentage": 98.97, "elapsed_time": "1 day, 11:57:33", "remaining_time": "0:22:26"}
+{"current_steps": 35400, "total_steps": 35758, "loss": 0.0067, "lr": 3.053061437530702e-09, "epoch": 1.9799211387343045, "percentage": 99.0, "elapsed_time": "1 day, 11:58:09", "remaining_time": "0:21:49"}
+{"current_steps": 35410, "total_steps": 35758, "loss": 0.0062, "lr": 2.884897672815745e-09, "epoch": 1.9804804384910093, "percentage": 99.03, "elapsed_time": "1 day, 11:58:47", "remaining_time": "0:21:12"}
+{"current_steps": 35420, "total_steps": 35758, "loss": 0.0064, "lr": 2.7214959477045446e-09, "epoch": 1.9810397382477138, "percentage": 99.05, "elapsed_time": "1 day, 11:59:24", "remaining_time": "0:20:36"}
+{"current_steps": 35430, "total_steps": 35758, "loss": 0.0068, "lr": 2.562856417910875e-09, "epoch": 1.9815990380044184, "percentage": 99.08, "elapsed_time": "1 day, 12:00:03", "remaining_time": "0:19:59"}
+{"current_steps": 35440, "total_steps": 35758, "loss": 0.0061, "lr": 2.4089792346132514e-09, "epoch": 1.9821583377611232, "percentage": 99.11, "elapsed_time": "1 day, 12:00:36", "remaining_time": "0:19:23"}
+{"current_steps": 35450, "total_steps": 35758, "loss": 0.0078, "lr": 2.2598645444488198e-09, "epoch": 1.9827176375178277, "percentage": 99.14, "elapsed_time": "1 day, 12:01:13", "remaining_time": "0:18:46"}
+{"current_steps": 35460, "total_steps": 35758, "loss": 0.0063, "lr": 2.1155124895178013e-09, "epoch": 1.9832769372745322, "percentage": 99.17, "elapsed_time": "1 day, 12:01:49", "remaining_time": "0:18:10"}
+{"current_steps": 35470, "total_steps": 35758, "loss": 0.0062, "lr": 1.9759232073818244e-09, "epoch": 1.983836237031237, "percentage": 99.19, "elapsed_time": "1 day, 12:02:25", "remaining_time": "0:17:33"}
+{"current_steps": 35480, "total_steps": 35758, "loss": 0.0064, "lr": 1.8410968310633714e-09, "epoch": 1.9843955367879413, "percentage": 99.22, "elapsed_time": "1 day, 12:03:00", "remaining_time": "0:16:56"}
+{"current_steps": 35490, "total_steps": 35758, "loss": 0.006, "lr": 1.7110334890468872e-09, "epoch": 1.984954836544646, "percentage": 99.25, "elapsed_time": "1 day, 12:03:35", "remaining_time": "0:16:20"}
+{"current_steps": 35500, "total_steps": 35758, "loss": 0.0055, "lr": 1.58573330527656e-09, "epoch": 1.9855141363013509, "percentage": 99.28, "elapsed_time": "1 day, 12:04:11", "remaining_time": "0:15:43"}
+{"current_steps": 35510, "total_steps": 35758, "loss": 0.0064, "lr": 1.4651963991585416e-09, "epoch": 1.9860734360580552, "percentage": 99.31, "elapsed_time": "1 day, 12:04:53", "remaining_time": "0:15:07"}
+{"current_steps": 35520, "total_steps": 35758, "loss": 0.0088, "lr": 1.3494228855592816e-09, "epoch": 1.98663273581476, "percentage": 99.33, "elapsed_time": "1 day, 12:05:29", "remaining_time": "0:14:30"}
+{"current_steps": 35530, "total_steps": 35758, "loss": 0.0049, "lr": 1.2384128748060832e-09, "epoch": 1.9871920355714645, "percentage": 99.36, "elapsed_time": "1 day, 12:06:08", "remaining_time": "0:13:54"}
+{"current_steps": 35540, "total_steps": 35758, "loss": 0.0059, "lr": 1.1321664726865466e-09, "epoch": 1.987751335328169, "percentage": 99.39, "elapsed_time": "1 day, 12:06:46", "remaining_time": "0:13:17"}
+{"current_steps": 35550, "total_steps": 35758, "loss": 0.0068, "lr": 1.0306837804496817e-09, "epoch": 1.9883106350848738, "percentage": 99.42, "elapsed_time": "1 day, 12:07:25", "remaining_time": "0:12:40"}
+{"current_steps": 35560, "total_steps": 35758, "loss": 0.0061, "lr": 9.3396489480313e-10, "epoch": 1.9888699348415784, "percentage": 99.45, "elapsed_time": "1 day, 12:07:56", "remaining_time": "0:12:04"}
+{"current_steps": 35570, "total_steps": 35758, "loss": 0.0068, "lr": 8.420099079170519e-10, "epoch": 1.989429234598283, "percentage": 99.47, "elapsed_time": "1 day, 12:08:30", "remaining_time": "0:11:27"}
+{"current_steps": 35580, "total_steps": 35758, "loss": 0.0041, "lr": 7.548189074196854e-10, "epoch": 1.9899885343549877, "percentage": 99.5, "elapsed_time": "1 day, 12:09:02", "remaining_time": "0:10:51"}
+{"current_steps": 35590, "total_steps": 35758, "loss": 0.0064, "lr": 6.723919764012321e-10, "epoch": 1.990547834111692, "percentage": 99.53, "elapsed_time": "1 day, 12:09:36", "remaining_time": "0:10:14"}
+{"current_steps": 35600, "total_steps": 35758, "loss": 0.006, "lr": 5.947291934105259e-10, "epoch": 1.9911071338683968, "percentage": 99.56, "elapsed_time": "1 day, 12:10:16", "remaining_time": "0:09:37"}
+{"current_steps": 35610, "total_steps": 35758, "loss": 0.0054, "lr": 5.218306324572542e-10, "epoch": 1.9916664336251013, "percentage": 99.59, "elapsed_time": "1 day, 12:10:51", "remaining_time": "0:09:01"}
+{"current_steps": 35620, "total_steps": 35758, "loss": 0.0071, "lr": 4.53696363010292e-10, "epoch": 1.9922257333818059, "percentage": 99.61, "elapsed_time": "1 day, 12:11:31", "remaining_time": "0:08:24"}
+{"current_steps": 35630, "total_steps": 35758, "loss": 0.0069, "lr": 3.903264499988124e-10, "epoch": 1.9927850331385106, "percentage": 99.64, "elapsed_time": "1 day, 12:12:07", "remaining_time": "0:07:48"}
+{"current_steps": 35640, "total_steps": 35758, "loss": 0.0063, "lr": 3.3172095381173166e-10, "epoch": 1.9933443328952152, "percentage": 99.67, "elapsed_time": "1 day, 12:12:43", "remaining_time": "0:07:11"}
+{"current_steps": 35650, "total_steps": 35758, "loss": 0.0059, "lr": 2.7787993029770863e-10, "epoch": 1.9939036326519197, "percentage": 99.7, "elapsed_time": "1 day, 12:13:21", "remaining_time": "0:06:35"}
+{"current_steps": 35660, "total_steps": 35758, "loss": 0.0083, "lr": 2.2880343076514543e-10, "epoch": 1.9944629324086245, "percentage": 99.73, "elapsed_time": "1 day, 12:13:58", "remaining_time": "0:05:58"}
+{"current_steps": 35670, "total_steps": 35758, "loss": 0.0063, "lr": 1.844915019810767e-10, "epoch": 1.995022232165329, "percentage": 99.75, "elapsed_time": "1 day, 12:14:33", "remaining_time": "0:05:21"}
+{"current_steps": 35680, "total_steps": 35758, "loss": 0.0043, "lr": 1.4494418617394535e-10, "epoch": 1.9955815319220336, "percentage": 99.78, "elapsed_time": "1 day, 12:15:07", "remaining_time": "0:04:45"}
+{"current_steps": 35690, "total_steps": 35758, "loss": 0.0069, "lr": 1.1016152102971689e-10, "epoch": 1.9961408316787383, "percentage": 99.81, "elapsed_time": "1 day, 12:15:45", "remaining_time": "0:04:08"}
+{"current_steps": 35700, "total_steps": 35758, "loss": 0.0046, "lr": 8.014353969576505e-11, "epoch": 1.9967001314354427, "percentage": 99.84, "elapsed_time": "1 day, 12:16:19", "remaining_time": "0:03:32"}
+{"current_steps": 35710, "total_steps": 35758, "loss": 0.007, "lr": 5.489027077698606e-11, "epoch": 1.9972594311921474, "percentage": 99.87, "elapsed_time": "1 day, 12:16:57", "remaining_time": "0:02:55"}
+{"current_steps": 35720, "total_steps": 35758, "loss": 0.006, "lr": 3.440173833968441e-11, "epoch": 1.997818730948852, "percentage": 99.89, "elapsed_time": "1 day, 12:17:33", "remaining_time": "0:02:18"}
+{"current_steps": 35730, "total_steps": 35758, "loss": 0.0058, "lr": 1.8677961907687115e-11, "epoch": 1.9983780307055565, "percentage": 99.92, "elapsed_time": "1 day, 12:18:09", "remaining_time": "0:01:42"}
+{"current_steps": 35740, "total_steps": 35758, "loss": 0.0064, "lr": 7.71895646511922e-12, "epoch": 1.9989373304622613, "percentage": 99.95, "elapsed_time": "1 day, 12:18:45", "remaining_time": "0:01:05"}
+{"current_steps": 35750, "total_steps": 35758, "loss": 0.0064, "lr": 1.5247324558487208e-12, "epoch": 1.9994966302189658, "percentage": 99.98, "elapsed_time": "1 day, 12:19:23", "remaining_time": "0:00:29"}
+{"current_steps": 35758, "total_steps": 35758, "epoch": 1.9999440700243296, "percentage": 100.0, "elapsed_time": "1 day, 12:20:18", "remaining_time": "0:00:00"}