diff --git "a/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" "b/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" --- "a/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" +++ "b/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" @@ -11529,3 +11529,791 @@ 2022-01-29 18:34:17,849 DEBUG SenderThread:13445 [sender.py:send():234] send: stats 2022-01-29 18:34:29,684 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status 2022-01-29 18:34:29,684 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:34:44,915 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:34:44,916 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:34:47,332 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:34:48,100 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:34:49,333 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:34:51,334 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:34:53,335 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:34:57,338 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:34:59,339 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:00,206 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:35:00,207 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:35:03,343 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:08,347 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:12,350 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:14,351 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:15,476 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:35:15,476 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:35:18,332 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:35:18,353 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:22,355 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:24,356 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:28,357 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:30,359 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:30,779 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:35:30,780 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:35:34,362 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:38,365 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:40,366 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:44,369 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:46,112 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:35:46,113 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:35:46,370 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:48,372 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:48,565 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:35:52,374 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:54,376 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:35:58,379 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:01,306 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:36:01,306 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:36:01,381 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:05,383 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:07,384 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:09,386 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:11,387 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:15,390 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:16,575 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:36:16,576 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:36:17,391 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:18,806 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:36:19,393 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:23,395 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:25,397 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:27,398 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:29,400 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:31,401 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:31,798 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:36:31,799 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:36:35,404 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:37,406 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:39,407 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:41,409 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:43,410 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:45,411 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:47,041 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:36:47,041 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:36:47,413 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:49,070 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:36:49,415 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:51,416 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:53,418 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:55,419 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:57,420 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:36:59,421 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:01,423 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:02,225 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:37:02,225 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:37:03,424 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:05,426 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:09,428 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:12,430 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:16,432 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:17,388 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:37:17,389 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:37:19,316 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:37:20,435 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:22,437 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:26,439 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:30,442 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:32,443 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:32,679 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:37:32,680 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:37:36,446 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:38,447 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:42,450 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:44,451 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:48,021 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:37:48,022 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:37:48,454 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:49,545 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:37:50,455 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:55,458 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:37:57,460 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:01,462 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:03,344 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:38:03,344 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:38:03,464 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:07,466 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:09,467 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:11,468 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:15,471 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:17,472 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:18,513 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:38:18,513 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:38:19,763 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:38:21,475 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:23,477 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:25,478 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:27,479 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:31,482 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:33,483 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:33,677 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:38:33,678 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:38:35,485 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:37,486 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:41,489 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:43,490 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:45,491 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:47,493 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:48,922 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:38:48,922 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:38:49,494 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:50,012 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:38:51,495 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:56,499 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:38:58,500 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:00,501 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:02,503 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:04,283 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:39:04,284 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:39:04,504 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:06,505 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:08,506 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:09,458 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 18:39:09,527 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 18:39:09,572 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 18:39:10,539 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:10,539 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 18:39:12,540 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:14,541 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:16,543 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:19,627 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:39:19,628 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:39:20,246 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:39:20,546 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:24,548 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:28,551 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:30,552 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:34,554 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:34,926 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:39:34,926 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:39:38,557 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:40,558 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:45,561 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:49,564 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:50,253 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:39:50,253 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:39:50,524 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:39:51,566 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:55,568 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:39:57,570 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:01,573 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:03,574 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:05,576 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:40:05,577 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:40:07,577 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:09,578 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:13,580 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:15,582 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:17,583 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:20,794 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:40:20,913 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:40:20,914 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:40:21,586 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:23,587 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:27,590 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:29,591 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:31,593 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:35,595 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:36,195 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:40:36,196 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:40:37,597 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:39,598 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:41,599 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:46,603 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:48,604 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:50,605 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:51,067 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:40:51,533 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:40:51,534 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:40:52,607 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:54,608 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:40:58,611 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:00,612 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:02,614 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:04,615 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:06,617 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:06,763 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:41:06,763 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:41:08,618 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:10,619 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:12,621 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:14,622 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:16,624 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:18,625 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:20,627 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:21,306 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:41:22,072 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:41:22,072 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:41:22,628 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:24,629 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:26,631 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:28,633 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:32,636 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:36,640 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:37,278 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:41:37,278 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:41:39,642 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:43,645 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:47,648 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:49,650 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:51,562 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:41:52,560 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:41:52,561 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:41:53,652 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:57,655 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:41:59,656 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:03,659 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:05,661 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:07,957 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:42:07,957 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:42:09,664 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:11,666 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:15,669 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:17,671 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:21,675 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:21,895 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:42:23,314 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:42:23,315 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:42:23,677 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:25,679 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:29,682 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:32,685 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:36,688 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:38,517 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:42:38,517 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:42:38,693 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:40,695 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:44,697 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:46,699 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:48,701 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:52,200 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:42:52,704 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:53,704 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:42:53,705 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:42:54,711 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:56,712 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:42:58,714 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:02,716 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:04,718 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:06,719 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:08,720 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:08,876 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:43:08,876 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:43:10,721 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:12,723 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:14,725 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:18,728 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:20,730 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:22,443 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:43:22,732 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:24,056 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:43:24,056 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:43:24,734 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:26,735 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:29,738 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:31,740 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:32,130 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 18:43:32,231 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 18:43:32,267 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 18:43:32,741 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 18:43:33,743 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:35,745 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:37,746 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:39,275 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:43:39,276 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:43:39,749 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:43,752 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:47,755 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:49,757 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:52,670 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:43:53,760 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:54,546 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:43:54,546 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:43:57,763 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:43:59,765 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:03,767 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:07,770 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:09,772 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:09,782 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:44:09,783 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:44:13,775 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:15,776 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:19,779 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:21,780 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:22,919 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:44:25,055 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:44:25,055 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:44:25,783 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:27,784 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:32,787 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:34,788 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:38,791 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:40,450 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:44:40,451 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:44:40,793 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:44,795 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:46,796 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:48,798 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:52,801 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:53,126 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:44:54,803 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:44:55,644 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:44:55,644 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:44:58,806 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:00,809 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:02,811 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:04,813 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:08,816 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:10,811 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:45:10,811 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:45:10,818 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:12,820 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:14,822 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:18,825 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:20,826 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:22,828 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:23,381 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:45:24,829 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:26,044 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:45:26,044 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:45:26,830 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:28,831 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:31,833 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:33,834 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:35,836 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:37,838 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:39,839 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:41,292 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:45:41,292 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:45:41,841 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:43,842 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:45,844 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:47,845 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:49,846 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:51,849 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:53,593 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:45:55,852 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:45:56,520 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:45:56,521 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:45:59,855 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:01,857 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:05,860 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:09,863 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:11,805 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:46:11,805 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:46:11,865 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:15,869 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:19,871 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:21,873 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:23,835 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:46:25,876 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:27,091 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:46:27,091 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:46:30,880 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:32,882 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:36,886 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:38,888 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:42,258 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:46:42,259 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:46:42,891 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:44,892 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:48,896 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:50,897 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:54,046 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:46:54,901 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:56,903 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:46:57,427 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:46:57,428 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:46:58,905 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:02,908 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:04,910 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:06,912 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:10,915 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:12,664 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:47:12,664 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:47:12,917 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:14,919 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:18,922 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:20,924 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:22,926 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:24,302 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:47:24,928 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:26,930 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:27,836 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:47:27,836 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:47:30,933 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:32,935 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:34,937 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:36,939 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:38,941 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:40,942 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:42,943 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:43,007 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:47:43,007 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:47:44,944 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:46,946 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:48,947 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:50,950 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:52,952 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:54,600 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:47:54,954 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:55,452 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 18:47:55,580 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 18:47:55,618 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 18:47:55,955 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 18:47:56,957 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:47:58,171 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:47:58,172 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:47:58,958 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:00,960 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:02,962 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:05,965 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:09,968 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:13,414 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:48:13,415 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:48:13,971 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:17,974 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:19,976 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:23,980 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:24,847 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:48:27,983 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:28,644 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:48:28,644 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:48:31,986 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:33,988 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:37,990 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:39,992 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:43,843 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:48:43,843 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:48:43,994 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:45,996 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:49,998 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:52,000 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:55,096 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:48:56,003 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:58,005 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:48:59,065 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:48:59,066 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:49:03,009 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:05,011 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:09,014 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:11,016 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:13,018 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:14,241 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:49:14,242 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:49:17,021 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:19,023 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:23,026 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:25,028 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:25,353 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:49:27,030 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:29,474 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:49:29,474 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:49:31,033 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:33,035 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:35,037 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:37,039 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:39,041 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:43,044 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:44,672 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:49:44,672 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:49:45,046 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:47,048 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:49,050 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:51,052 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:53,054 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:55,056 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:55,595 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:49:57,058 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:59,060 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:49:59,840 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:49:59,841 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:50:01,062 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:03,063 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:05,065 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:07,067 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:09,069 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:11,071 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:14,074 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:15,051 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:50:15,052 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:50:16,076 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:18,078 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:22,081 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:25,899 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:50:26,084 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:28,086 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:30,245 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:50:30,246 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:50:32,089 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:36,092 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:38,094 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:42,097 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:45,506 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:50:45,506 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:50:46,100 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:48,102 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:52,106 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:54,108 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:50:56,164 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:50:58,111 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:00,113 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:00,694 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:51:00,695 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:51:04,116 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:06,118 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:10,121 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:12,123 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:15,878 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:51:15,879 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:51:16,126 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:18,128 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:20,130 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:24,134 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:26,136 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:26,425 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:51:30,139 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:31,124 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:51:31,124 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:51:32,141 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:35,143 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:37,145 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:41,149 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:43,150 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:45,151 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:46,457 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:51:46,457 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:51:47,153 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:51,156 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:53,157 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:55,159 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:56,714 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:51:57,160 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:51:59,161 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:01,162 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:01,723 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:52:01,723 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:52:03,164 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:05,166 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:09,168 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:11,170 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:13,171 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:15,173 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:16,982 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:52:16,982 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:52:17,174 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:18,175 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 18:52:18,248 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 18:52:18,296 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 18:52:18,297 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 18:52:19,297 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:21,298 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:23,300 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:25,302 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:26,926 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:52:29,305 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:32,266 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:52:32,267 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:52:33,309 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:35,310 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:40,314 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:44,317 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:46,319 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:47,540 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:52:47,541 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:52:50,322 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:54,325 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:56,327 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:52:57,200 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:53:00,331 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:02,333 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:02,712 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:53:02,712 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:53:06,337 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:08,339 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:12,342 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:14,345 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:17,878 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:53:17,879 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:53:18,348 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:20,350 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:24,353 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:26,355 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:27,479 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:53:28,357 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:32,360 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:33,039 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:53:33,040 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:53:34,362 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:38,365 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:40,367 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:42,370 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:46,373 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:48,236 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:53:48,236 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:53:48,374 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:50,377 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:52,378 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:55,380 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:53:57,796 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:53:59,384 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:01,386 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:03,387 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:03,404 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:54:03,405 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:54:05,389 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:07,391 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:09,393 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:13,396 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:15,398 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:17,400 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:18,580 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:54:18,581 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:54:19,402 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:21,403 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:23,404 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:25,405 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:27,407 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:28,173 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:54:29,408 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:31,410 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:33,411 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:33,802 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:54:33,802 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:54:35,413 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:37,414 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:39,416 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:43,418 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:47,422 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:49,040 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:54:49,040 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:54:49,423 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:53,426 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:57,428 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:54:58,461 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:54:59,430 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:03,433 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:04,331 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:55:04,332 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:55:07,436 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:09,437 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:13,439 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:16,441 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:19,491 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:55:19,491 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:55:20,443 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:22,444 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:26,447 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:28,449 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:28,705 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:55:30,450 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:34,453 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:34,660 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:55:34,661 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:55:36,455 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:40,458 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:42,460 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:44,461 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:48,465 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:49,916 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:55:49,916 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:55:50,466 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:52,468 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:56,471 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:58,474 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:55:59,058 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:56:00,476 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:02,477 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:05,297 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:56:05,298 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:56:06,481 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:08,483 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:10,485 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:12,487 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:15,489 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:19,492 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:20,543 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:56:20,544 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:56:21,494 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:23,496 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:25,498 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:27,500 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:29,371 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:56:29,502 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:31,504 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:33,505 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:35,507 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:35,784 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:56:35,785 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:56:37,509 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:39,109 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 18:56:39,209 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 18:56:39,261 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 18:56:39,511 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:39,512 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 18:56:40,513 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:41,514 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:43,516 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:45,518 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:47,520 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:49,522 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:51,017 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:56:51,018 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:56:51,525 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:53,526 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:55,528 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:57,530 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:59,532 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:56:59,581 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:57:01,534 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:03,536 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:05,538 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:06,270 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:57:06,271 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:57:07,540 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:09,542 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:11,544 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:13,546 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:15,548 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:17,550 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:19,552 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:21,534 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:57:21,535 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:57:21,553 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:23,554 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:25,555 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:27,556 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:29,558 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:29,791 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:57:31,559 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:33,560 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:35,562 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:36,800 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:57:36,801 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:57:37,563 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:43,197 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 18:57:43,232 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 18:57:43,280 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 18:57:43,567 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 18:57:44,569 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:45,570 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:47,572 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:57:52,013 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:57:52,015 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:57:58,579 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:58:00,011 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:58:07,185 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:58:07,186 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:58:22,350 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:58:22,351 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:58:30,218 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:58:37,524 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:58:37,525 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:58:52,699 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:58:52,700 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:59:00,422 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:59:07,870 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:59:07,871 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status