diff --git "a/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" "b/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" --- "a/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" +++ "b/wandb/run-20220129_131141-h6nhqm30/logs/debug-internal.log" @@ -12317,3 +12317,795 @@ 2022-01-29 18:59:00,422 DEBUG SenderThread:13445 [sender.py:send():234] send: stats 2022-01-29 18:59:07,870 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status 2022-01-29 18:59:07,871 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:59:23,047 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:59:23,047 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:59:30,637 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:30,643 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 18:59:32,638 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:34,640 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:36,641 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:38,216 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:59:38,217 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:59:41,645 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:45,648 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:47,649 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:51,652 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:53,485 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 18:59:53,486 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 18:59:55,654 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 18:59:57,655 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:00,897 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:00:01,658 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:03,659 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:07,662 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:08,750 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:00:08,750 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:00:11,664 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:13,666 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:17,668 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:19,670 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:23,672 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:23,971 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:00:23,971 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:00:25,674 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:29,677 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:31,136 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:00:31,679 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:35,681 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:38,683 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:39,146 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:00:39,147 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:00:40,685 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:44,687 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:46,689 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:50,691 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:52,693 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:54,519 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:00:54,520 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:00:54,694 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:00:58,696 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:00,698 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:01,368 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:01:02,699 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:04,701 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:08,703 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:09,818 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:01:09,818 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:01:10,704 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:12,706 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:14,707 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:16,709 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:20,711 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:22,713 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:24,714 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:25,079 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:01:25,080 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:01:26,716 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:28,717 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:30,718 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:31,590 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:01:32,719 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:34,720 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:36,722 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:38,723 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:40,301 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:01:40,301 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:01:40,725 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:42,726 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:44,727 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:46,729 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:48,730 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:52,732 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:55,550 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:01:55,551 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:01:55,739 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:01:59,742 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:01,808 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:02:03,745 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:05,746 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:09,748 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:10,853 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:02:10,854 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:02:13,751 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:15,752 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:19,755 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:23,758 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:25,759 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:26,032 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:02:26,033 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:02:29,762 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:31,764 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:32,051 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:02:35,767 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:37,769 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:41,230 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:02:41,230 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:02:41,772 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:43,774 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:47,777 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:49,780 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:51,782 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:55,785 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:56,401 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:02:56,402 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:02:57,787 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:02:59,789 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:02,284 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:03:03,792 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:06,794 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:08,796 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:11,570 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:03:11,570 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:03:12,799 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:14,802 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:16,803 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:18,805 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:22,808 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:24,810 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:26,752 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:03:26,752 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:03:26,812 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:28,814 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:30,816 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:32,528 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:03:34,819 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:36,821 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:38,823 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:40,825 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:41,945 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:03:41,945 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:03:42,827 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:44,829 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:46,831 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:48,832 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:50,834 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:52,836 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:53,030 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:03:53,101 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:03:53,149 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:03:53,838 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:03:54,839 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:55,840 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:56,842 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:03:57,110 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:03:57,111 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:03:58,844 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:00,845 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:02,851 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:04:04,847 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:08,850 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:10,851 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:12,273 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:04:12,274 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:04:14,854 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:16,856 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:20,858 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:25,862 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:27,533 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:04:27,533 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:04:27,864 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:31,867 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:33,076 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:04:33,870 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:37,873 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:41,876 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:42,940 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:04:42,940 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:04:43,878 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:47,881 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:49,883 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:53,886 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:55,888 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:04:58,352 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:04:58,352 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:04:59,891 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:01,893 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:03,404 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:05:05,896 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:07,899 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:09,900 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:13,683 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:05:13,683 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:05:13,903 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:15,904 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:18,906 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:22,909 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:24,910 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:26,912 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:28,894 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:05:28,894 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:05:28,913 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:32,916 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:33,673 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:05:34,917 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:36,918 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:38,919 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:40,921 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:44,085 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:05:44,085 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:05:44,924 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:46,925 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:48,927 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:50,928 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:52,930 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:54,932 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:56,934 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:58,936 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:05:59,257 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:05:59,258 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:06:00,938 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:02,940 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:03,906 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:06:04,942 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:06,944 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:08,945 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:10,947 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:12,949 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:14,428 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:06:14,429 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:06:16,952 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:18,954 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:22,957 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:26,960 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:28,962 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:29,697 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:06:29,698 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:06:33,966 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:34,146 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:06:35,968 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:39,971 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:43,974 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:45,057 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:06:45,057 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:06:45,976 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:49,979 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:51,981 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:55,985 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:06:59,988 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:00,385 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:07:00,385 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:07:01,990 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:04,360 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:07:05,993 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:07,995 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:09,997 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:14,001 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:15,715 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:07:15,715 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:07:16,003 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:20,006 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:23,008 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:27,012 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:29,013 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:30,895 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:07:30,896 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:07:31,014 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:33,016 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:34,627 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:07:37,019 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:39,021 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:41,024 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:43,026 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:46,086 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:07:46,087 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:07:47,029 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:49,031 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:51,033 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:53,034 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:57,036 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:07:59,038 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:01,039 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:01,347 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:08:01,347 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:08:03,041 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:04,942 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:08:05,042 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:07,043 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:09,045 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:11,046 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:13,047 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:15,049 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:16,578 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:08:16,578 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:08:17,050 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:17,463 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:08:17,536 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:08:17,582 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:08:18,051 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:08:19,052 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:21,053 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:23,055 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:25,056 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:29,059 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:31,060 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:31,810 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:08:31,811 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:08:35,062 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:35,272 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:08:39,065 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:41,066 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:45,070 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:46,986 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:08:46,987 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:08:50,073 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:52,074 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:08:56,077 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:00,080 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:02,082 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:02,191 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:09:02,191 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:09:05,530 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:09:06,085 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:08,087 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:12,090 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:14,092 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:17,497 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:09:17,497 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:09:18,095 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:20,097 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:24,100 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:26,102 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:30,105 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:32,107 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:32,690 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:09:32,691 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:09:34,108 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:35,749 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:09:38,112 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:40,114 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:42,115 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:46,119 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:47,942 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:09:47,942 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:09:48,120 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:51,123 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:55,126 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:57,128 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:09:59,130 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:01,131 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:03,239 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:10:03,240 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:10:05,134 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:06,090 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:10:07,137 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:09,139 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:11,140 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:13,142 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:15,144 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:17,145 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:18,531 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:10:18,532 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:10:19,147 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:21,148 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:23,150 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:25,151 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:27,152 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:29,154 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:31,155 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:33,156 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:33,820 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:10:33,820 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:10:35,158 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:36,316 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:10:37,159 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:41,162 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:45,164 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:47,165 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:48,984 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:10:48,985 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:10:51,168 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:56,172 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:10:58,173 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:02,176 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:04,280 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:11:04,281 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:11:06,179 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:06,585 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:11:08,180 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:12,183 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:14,185 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:18,187 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:19,597 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:11:19,598 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:11:20,189 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:24,193 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:26,195 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:30,198 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:32,200 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:34,967 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:11:34,967 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:11:36,203 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:37,083 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:11:38,205 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:40,207 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:44,210 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:46,212 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:50,173 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:11:50,174 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:11:50,259 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:52,261 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:54,264 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:11:57,266 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:01,269 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:03,271 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:05,273 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:05,349 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:12:05,350 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:12:07,275 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:07,330 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:12:11,278 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:13,281 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:15,283 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:17,285 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:19,286 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:20,577 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:12:20,578 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:12:21,288 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:25,291 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:27,293 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:29,295 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:31,297 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:33,299 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:35,301 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:35,863 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:12:35,863 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:12:37,303 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:37,592 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:12:39,305 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:41,233 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:12:41,331 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:12:41,347 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:41,382 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:12:42,343 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:42,343 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:12:43,344 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:45,346 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:47,348 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:49,350 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:51,076 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:12:51,077 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:12:53,353 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:57,356 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:12:59,358 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:03,361 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:06,591 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:13:06,591 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:13:07,365 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:07,867 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:13:09,366 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:13,369 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:17,372 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:19,374 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:21,901 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:13:21,902 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:13:23,377 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:28,380 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:30,382 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:34,384 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:36,385 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:37,165 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:13:37,165 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:13:38,133 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:13:40,388 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:42,389 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:46,393 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:48,394 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:52,336 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:13:52,336 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:13:52,396 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:54,398 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:13:56,400 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:00,402 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:02,404 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:06,406 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:07,637 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:14:07,637 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:14:08,375 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:14:08,409 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:10,411 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:14,414 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:16,415 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:18,417 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:20,418 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:22,962 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:14:22,963 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:14:24,421 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:26,423 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:28,424 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:30,425 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:32,427 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:35,431 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:38,210 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:14:38,210 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:14:38,618 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:14:39,433 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:41,437 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:43,438 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:45,440 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:47,442 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:49,444 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:51,447 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:53,424 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:14:53,424 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:14:53,449 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:55,451 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:57,453 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:14:59,456 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:01,458 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:05,466 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:08,739 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:15:08,739 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:15:08,898 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:15:09,471 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:13,473 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:15,475 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:19,478 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:23,480 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:23,923 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:15:23,923 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:15:27,486 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:29,491 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:33,494 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:35,500 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:39,152 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:15:39,153 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:15:39,309 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:15:40,503 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:44,506 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:46,507 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:50,510 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:52,511 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:54,510 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:15:54,510 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:15:56,517 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:15:58,518 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:02,521 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:04,522 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:06,523 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:09,574 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:16:09,782 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:16:09,783 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:16:10,526 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:12,528 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:14,529 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:18,532 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:20,534 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:22,535 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:25,060 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:16:25,061 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:16:26,539 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:28,540 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:30,542 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:32,543 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:37,547 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:39,548 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:39,920 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:16:40,322 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:16:40,323 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:16:41,550 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:43,551 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:45,552 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:47,554 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:49,556 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:53,558 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:55,560 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:55,561 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:16:55,562 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:16:57,561 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:16:59,562 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:01,564 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:03,565 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:05,566 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:06,963 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:17:07,052 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:17:07,085 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:17:07,568 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:07,568 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:17:09,569 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:10,237 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:17:10,850 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:17:10,851 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:17:11,570 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:13,572 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:17,575 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:21,578 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:25,584 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:26,020 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:17:26,020 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:17:27,585 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:31,588 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:36,592 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:38,593 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:40,598 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:17:41,298 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:17:41,299 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:17:42,596 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:44,597 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:48,600 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:52,603 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:54,604 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:17:56,682 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:17:56,683 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:17:58,607 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:00,608 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:04,611 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:06,613 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:08,615 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:10,832 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:18:12,004 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:18:12,005 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:18:12,618 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:14,620 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:18,623 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:20,625 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:24,629 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:26,631 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:27,263 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:18:27,263 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:18:28,633 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:32,635 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:34,637 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:37,639 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:41,071 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:18:41,642 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:42,518 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:18:42,518 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:18:43,650 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:47,655 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:49,656 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:51,658 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:53,659 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:55,661 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:57,671 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:18:57,729 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:18:57,729 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:18:59,677 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:03,680 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:05,682 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:07,684 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:09,686 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:11,383 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:19:11,689 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:12,900 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:19:12,901 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:19:13,690 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:15,693 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:17,696 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:19,697 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:21,699 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:23,701 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:25,703 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:28,069 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:19:28,069 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:19:29,707 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:33,712 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:36,715 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:40,718 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:41,694 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:19:43,342 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:19:43,343 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:19:44,721 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:46,723 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:50,726 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:54,729 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:56,731 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:19:58,616 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:19:58,617 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:20:00,735 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:02,737 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:06,740 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:10,743 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:11,937 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:20:12,744 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:13,862 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:20:13,862 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:20:14,746 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:18,749 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:20,751 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:24,754 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:26,756 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:29,221 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:20:29,222 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:20:30,759 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:33,762 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:37,765 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:39,767 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:41,769 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:42,187 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:20:44,498 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:20:44,498 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:20:45,772 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:47,774 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:49,776 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:53,779 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:55,780 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:57,782 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:20:59,682 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:20:59,682 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:20:59,783 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:01,785 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:05,788 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:07,790 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:09,792 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:11,794 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:12,462 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:21:13,796 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:14,927 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:21:14,928 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:21:15,798 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:17,800 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:19,802 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:21,803 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:23,806 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:25,807 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:27,809 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:29,810 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:30,171 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:21:30,171 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:21:30,492 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:21:30,563 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:21:30,620 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:21:30,811 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:21:31,812 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:33,813 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:35,815 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:38,817 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:40,818 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:42,769 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:21:42,820 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:44,821 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:45,433 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:21:45,434 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:21:46,823 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:48,824 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:50,825 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:52,827 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:54,832 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:56,834 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:21:58,836 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:00,654 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:22:00,655 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:22:00,838 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:02,839 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:04,841 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:06,842 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:08,844 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:10,846 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:12,848 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:13,071 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:22:14,851 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:15,884 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:22:15,885 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:22:16,852 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:18,854 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:20,856 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:22,858 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:24,860 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:26,862 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:29,864 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:31,162 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:22:31,163 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:22:31,866 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:36,200 DEBUG SenderThread:13445 [sender.py:send():234] send: history +2022-01-29 19:22:36,239 DEBUG SenderThread:13445 [sender.py:send():234] send: summary +2022-01-29 19:22:36,292 INFO SenderThread:13445 [sender.py:_save_file():939] saving file wandb-summary.json with policy end +2022-01-29 19:22:36,870 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/wandb-summary.json +2022-01-29 19:22:37,871 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:41,874 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:22:43,310 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:22:46,353 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:22:46,354 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:22:51,881 INFO Thread-8 :13445 [dir_watcher.py:_on_file_modified():230] file/dir modified: /workspace/xls-r-300m-fr/wandb/run-20220129_131141-h6nhqm30/files/output.log +2022-01-29 19:23:01,743 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:23:01,744 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:23:13,526 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:23:16,915 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:23:16,916 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:23:32,296 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:23:32,297 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:23:43,783 DEBUG SenderThread:13445 [sender.py:send():234] send: stats +2022-01-29 19:23:47,462 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:23:47,463 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:24:02,639 DEBUG HandlerThread:13445 [handler.py:handle_request():130] handle_request: stop_status +2022-01-29 19:24:02,640 DEBUG SenderThread:13445 [sender.py:send_request():248] send_request: stop_status +2022-01-29 19:24:14,036 DEBUG SenderThread:13445 [sender.py:send():234] send: stats